From b235a919c92571ec9846f8ecb935d2afbcb66b69 Mon Sep 17 00:00:00 2001 From: shulan Date: Tue, 17 Dec 2024 23:23:11 +0800 Subject: [PATCH] feat: support invalidate module (#2005) * feat: support invalidate module * feat: remove unless code * feat: clear expired module --------- Co-authored-by: ADNY <66500121+ErKeLost@users.noreply.github.com> Co-authored-by: brightwu <1521488775@qq.com> --- .changeset/spicy-snakes-pay.md | 5 ++ crates/compiler/src/build/module_cache.rs | 1 + crates/core/src/cache/cache_store.rs | 15 +++++- crates/core/src/cache/module_cache.rs | 8 ++++ .../cache/module_cache/immutable_modules.rs | 47 ++++++++++++------- .../src/cache/module_cache/mutable_modules.rs | 24 ++++++---- crates/core/src/context/mod.rs | 4 ++ crates/node/src/lib.rs | 12 +++++ packages/core/binding/binding.d.ts | 1 + packages/core/src/compiler/index.ts | 4 ++ 10 files changed, 96 insertions(+), 25 deletions(-) create mode 100644 .changeset/spicy-snakes-pay.md diff --git a/.changeset/spicy-snakes-pay.md b/.changeset/spicy-snakes-pay.md new file mode 100644 index 000000000..cf1965731 --- /dev/null +++ b/.changeset/spicy-snakes-pay.md @@ -0,0 +1,5 @@ +--- +"@farmfe/core": patch +--- + +feat: support invalidateModule diff --git a/crates/compiler/src/build/module_cache.rs b/crates/compiler/src/build/module_cache.rs index 6db300edf..baf558b74 100644 --- a/crates/compiler/src/build/module_cache.rs +++ b/crates/compiler/src/build/module_cache.rs @@ -234,6 +234,7 @@ pub fn set_module_graph_cache(module_ids: Vec, context: &Arc, pub watch_dependencies: Vec, + /// + /// `default`: false + /// + /// true: it makes the cache expire. + /// + /// when writing to the cache next time, it will be cleared from memory. + /// + pub is_expired: bool, } impl CachedModule { diff --git a/crates/core/src/cache/module_cache/immutable_modules.rs b/crates/core/src/cache/module_cache/immutable_modules.rs index f4a1722f8..7560f87e9 100644 --- a/crates/core/src/cache/module_cache/immutable_modules.rs +++ b/crates/core/src/cache/module_cache/immutable_modules.rs @@ -104,15 +104,7 @@ impl ImmutableModulesMemoryStore { impl ModuleMemoryStore for ImmutableModulesMemoryStore { fn has_cache(&self, key: &crate::module::ModuleId) -> bool { - if self.cached_modules.contains_key(key) { - return true; - } - - if let Some(package_key) = self.manifest.get(key) { - return self.store.has_cache(package_key.value()); - } - - false + self.get_cache_ref(key).is_some_and(|m| !m.is_expired) } fn set_cache(&self, key: crate::module::ModuleId, module: super::CachedModule) { @@ -179,18 +171,45 @@ impl ModuleMemoryStore for ImmutableModulesMemoryStore { fn write_cache(&self) { let mut packages = HashMap::new(); + let mut pending_remove_modules = HashSet::new(); + let mut maybe_remove_package = HashSet::new(); for item in self.cached_modules.iter() { let module = item.value(); + let package_key = CachedPackage::gen_key(&module.module.package_name, &module.module.package_version); + if module.is_expired { + pending_remove_modules.insert(item.key().clone()); + maybe_remove_package.insert(package_key); + continue; + } + let package = packages.entry(package_key.clone()).or_insert_with(Vec::new); package.push(item.key().clone()); self.manifest.insert(item.key().clone(), package_key); } + for key in pending_remove_modules { + self.cached_modules.remove(&key); + self.manifest.remove(&key); + self.manifest_reversed.iter_mut().for_each(|mut item| { + if item.value_mut().contains(&key) { + item.value_mut().remove(&key); + } + }) + } + + for package in maybe_remove_package { + if packages.contains_key(&package) { + return; + } + + self.store.remove_cache(&package); + } + let manifest = self .manifest .iter() @@ -283,13 +302,9 @@ impl ModuleMemoryStore for ImmutableModulesMemoryStore { } fn invalidate_cache(&self, key: &ModuleId) { - self.cached_modules.remove(key); - self.manifest.remove(key); - self.manifest_reversed.iter_mut().for_each(|mut item| { - if item.value_mut().contains(key) { - item.value_mut().remove(key); - } - }) + if let Some(mut m) = self.get_cache_mut_ref(key) { + m.is_expired = true; + } } fn is_cache_changed(&self, module: &crate::module::Module) -> bool { diff --git a/crates/core/src/cache/module_cache/mutable_modules.rs b/crates/core/src/cache/module_cache/mutable_modules.rs index 882f75e0d..95ed0b4be 100644 --- a/crates/core/src/cache/module_cache/mutable_modules.rs +++ b/crates/core/src/cache/module_cache/mutable_modules.rs @@ -22,7 +22,7 @@ pub struct MutableModulesMemoryStore { /// ModuleId -> Cached Module cached_modules: DashMap, } - +// TODO: cache unit test impl MutableModulesMemoryStore { pub fn new(cache_dir_str: &str, namespace: &str, mode: Mode) -> Self { Self { @@ -58,11 +58,7 @@ impl MutableModulesMemoryStore { impl ModuleMemoryStore for MutableModulesMemoryStore { fn has_cache(&self, key: &ModuleId) -> bool { - if self.cached_modules.contains_key(key) { - return true; - } - - self.store.has_cache(&key.to_string()) + self.get_cache_ref(key).is_some_and(|m| !m.is_expired) } fn set_cache(&self, key: ModuleId, module: CachedModule) { @@ -125,9 +121,14 @@ impl ModuleMemoryStore for MutableModulesMemoryStore { fn write_cache(&self) { let mut cache_map = HashMap::new(); - + let mut pending_removed_modules = vec![]; for entry in self.cached_modules.iter() { let module = entry.value(); + if module.is_expired { + pending_removed_modules.push(module.module.id.clone()); + continue; + } + let store_key = self.gen_cache_store_key(&module.module); if self.store.is_cache_changed(&store_key) { @@ -141,10 +142,17 @@ impl ModuleMemoryStore for MutableModulesMemoryStore { .collect::>(); self.store.write_cache(cache_map); + + for module_id in pending_removed_modules { + self.cached_modules.remove(&module_id); + self.store.remove_cache(&module_id.to_string()); + } } fn invalidate_cache(&self, key: &ModuleId) { - self.cached_modules.remove(key); + if let Some(mut m) = self.get_cache_mut_ref(key) { + m.is_expired = true; + }; } fn is_cache_changed(&self, module: &crate::module::Module) -> bool { diff --git a/crates/core/src/context/mod.rs b/crates/core/src/context/mod.rs index 2ff2c610b..a95bf0d34 100644 --- a/crates/core/src/context/mod.rs +++ b/crates/core/src/context/mod.rs @@ -164,6 +164,10 @@ impl CompilationContext { resolve_cache.insert(param, result); } + pub fn invalidate_module(&self, module_id: &ModuleId) { + self.cache_manager.module_cache.invalidate_cache(module_id); + } + pub fn clear_log_store(&self) { let mut log_store = self.log_store.lock(); log_store.clear(); diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index 980a7e8cf..e9973fe14 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -475,6 +475,18 @@ impl JsCompiler { let context = self.compiler.context(); context.record_manager.to_string() } + + #[napi] + pub fn invalidate_module(&self, module_id: String) { + invalidate_module(self, module_id); + } +} + +fn invalidate_module(js_compiler: &JsCompiler, module_id: String) { + let context = js_compiler.compiler.context(); + let module_id = ModuleId::new(&module_id, "", &context.config.root); + + context.invalidate_module(&module_id); } #[cfg(feature = "file_watcher")] diff --git a/packages/core/binding/binding.d.ts b/packages/core/binding/binding.d.ts index eff15ee97..fe777018a 100644 --- a/packages/core/binding/binding.d.ts +++ b/packages/core/binding/binding.d.ts @@ -78,4 +78,5 @@ export declare class Compiler { relativeModulePaths(): Array resource(name: string): Buffer | null stats(): string + invalidateModule(moduleId: string): void } diff --git a/packages/core/src/compiler/index.ts b/packages/core/src/compiler/index.ts index 51788ab04..f5b930f58 100644 --- a/packages/core/src/compiler/index.ts +++ b/packages/core/src/compiler/index.ts @@ -240,4 +240,8 @@ export class Compiler { stats() { return this._bindingCompiler.stats(); } + + invalidateModule(moduleId: string) { + this._bindingCompiler.invalidateModule(moduleId); + } }