feat: lazy compilation

crates/node_binding/binding.d.ts

@@ -78,7 +78,7 @@ export class Rspack {
    * Warning:
    * Calling this method recursively will cause a deadlock.
    */
-  unsafe_rebuild(changed_files: string[], removed_files: string[], callback: (err: null | Error) => void): void
+  unsafe_rebuild(changed_files: string[], removed_files: string[], modified_modules: string[], callback: (err: null | Error) => void): void
   /**
    * Get the last compilation
    *
@@ -197,7 +197,8 @@ export enum BuiltinPluginName {
   HtmlRspackPlugin = 'HtmlRspackPlugin',
   SwcJsMinimizerRspackPlugin = 'SwcJsMinimizerRspackPlugin',
   SwcCssMinimizerRspackPlugin = 'SwcCssMinimizerRspackPlugin',
-  BundlerInfoRspackPlugin = 'BundlerInfoRspackPlugin'
+  BundlerInfoRspackPlugin = 'BundlerInfoRspackPlugin',
+  LazyCompilation = 'LazyCompilation'
 }
 
 export function cleanupGlobalTrace(): void
@@ -912,6 +913,14 @@ export interface RawJavascriptParserOptions {
   url: string
 }
 
+export interface RawLazyCompilationOption {
+  module: (...args: any[]) => any
+  dispose: (...args: any[]) => any
+  test?: RawRegexMatcher
+  entries: boolean
+  imports: boolean
+}
+
 export interface RawLibraryAuxiliaryComment {
   root?: string
   commonjs?: string
@@ -947,6 +956,11 @@ export interface RawLimitChunkCountPluginOptions {
   maxChunks: number
 }
 
+export interface RawModuleArg {
+  module: string
+  path: string
+}
+
 export interface RawModuleFilenameTemplateFnCtx {
   identifier: string
   shortIdentifier: string
@@ -961,6 +975,12 @@ export interface RawModuleFilenameTemplateFnCtx {
   namespace: string
 }
 
+export interface RawModuleInfo {
+  active: boolean
+  client: string
+  data: string
+}
+
 export interface RawModuleOptions {
   rules: Array<RawModuleRule>
   parser?: Record<string, RawParserOptions>

crates/node_binding/src/lib.rs

@@ -148,13 +148,14 @@ impl Rspack {
   /// Calling this method recursively will cause a deadlock.
   #[napi(
     js_name = "unsafe_rebuild",
-    ts_args_type = "changed_files: string[], removed_files: string[], callback: (err: null | Error) => void"
+    ts_args_type = "changed_files: string[], removed_files: string[], modified_modules: string[], callback: (err: null | Error) => void"
   )]
   pub fn rebuild(
     &self,
     env: Env,
     changed_files: Vec<String>,
     removed_files: Vec<String>,
+    force_build_modules: Vec<String>,
     f: JsFunction,
   ) -> Result<()> {
     let handle_rebuild = |compiler: &mut Pin<Box<rspack_core::Compiler<_>>>| {
@@ -170,6 +171,7 @@ impl Rspack {
           .rebuild(
             HashSet::from_iter(changed_files.into_iter()),
             HashSet::from_iter(removed_files.into_iter()),
+            HashSet::from_iter(force_build_modules.into_iter()),
           )
           .await
           .map_err(|e| {

crates/rspack_binding_options/Cargo.toml

@@ -38,6 +38,7 @@ rspack_plugin_hmr                     = { path = "../rspack_plugin_hmr" }
 rspack_plugin_html                    = { path = "../rspack_plugin_html" }
 rspack_plugin_javascript              = { path = "../rspack_plugin_javascript" }
 rspack_plugin_json                    = { path = "../rspack_plugin_json" }
+rspack_plugin_lazy_compilation        = { path = "../rspack_plugin_lazy_compilation" }
 rspack_plugin_library                 = { path = "../rspack_plugin_library" }
 rspack_plugin_limit_chunk_count       = { path = "../rspack_plugin_limit_chunk_count" }
 rspack_plugin_merge_duplicate_chunks  = { path = "../rspack_plugin_merge_duplicate_chunks" }
@@ -56,10 +57,11 @@ rspack_plugin_web_worker_template     = { path = "../rspack_plugin_web_worker_te
 rspack_plugin_worker                  = { path = "../rspack_plugin_worker" }
 rspack_regex                          = { path = "../rspack_regex" }
 rspack_swc_visitors                   = { path = "../rspack_swc_visitors" }
-rustc-hash                            = { workspace = true }
-serde                                 = { workspace = true, features = ["derive"] }
-serde_json                            = { workspace = true }
-swc_config                            = { workspace = true }
-swc_core                              = { workspace = true, default-features = false, features = ["ecma_transforms_react"] }
-tokio                                 = { workspace = true, features = ["rt", "rt-multi-thread", "macros", "test-util", "parking_lot"] }
-tracing                               = { workspace = true }
+
+rustc-hash = { workspace = true }
+serde      = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+swc_config = { workspace = true }
+swc_core   = { workspace = true, default-features = false, features = ["ecma_transforms_react"] }
+tokio      = { workspace = true, features = ["rt", "rt-multi-thread", "macros", "test-util", "parking_lot"] }
+tracing    = { workspace = true }

crates/rspack_binding_options/src/options/raw_builtins/mod.rs

@@ -2,6 +2,7 @@ mod raw_banner;
 mod raw_bundle_info;
 mod raw_copy;
 mod raw_html;
+mod raw_lazy_compilation;
 mod raw_limit_chunk_count;
 mod raw_mf;
 mod raw_progress;
@@ -10,7 +11,7 @@ mod raw_to_be_deprecated;
 
 use napi::{bindgen_prelude::FromNapiValue, JsUnknown};
 use napi_derive::napi;
-use rspack_core::{BoxPlugin, Define, DefinePlugin, PluginExt, Provide, ProvidePlugin};
+use rspack_core::{BoxPlugin, Define, DefinePlugin, Plugin, PluginExt, Provide, ProvidePlugin};
 use rspack_error::Result;
 use rspack_ids::{
   DeterministicChunkIdsPlugin, DeterministicModuleIdsPlugin, NamedChunkIdsPlugin,
@@ -58,6 +59,7 @@ use rspack_plugin_warn_sensitive_module::WarnCaseSensitiveModulesPlugin;
 use rspack_plugin_wasm::{enable_wasm_loading_plugin, AsyncWasmPlugin};
 use rspack_plugin_web_worker_template::web_worker_template_plugin;
 use rspack_plugin_worker::WorkerPlugin;
+use rspack_regex::RspackRegex;
 
 pub use self::{
   raw_banner::RawBannerPluginOptions, raw_copy::RawCopyRspackPluginOptions,
@@ -67,6 +69,7 @@ pub use self::{
 };
 use self::{
   raw_bundle_info::{RawBundlerInfoModeWrapper, RawBundlerInfoPluginOptions},
+  raw_lazy_compilation::{JsBackend, RawLazyCompilationOption},
   raw_mf::{RawConsumeSharedPluginOptions, RawContainerReferencePluginOptions, RawProvideOptions},
 };
 use crate::{
@@ -138,6 +141,8 @@ pub enum BuiltinPluginName {
   SwcJsMinimizerRspackPlugin,
   SwcCssMinimizerRspackPlugin,
   BundlerInfoRspackPlugin,
+
+  LazyCompilation,
 }
 
 #[napi(object)]
@@ -391,6 +396,23 @@ impl BuiltinPlugin {
           .boxed(),
         )
       }
+      BuiltinPluginName::LazyCompilation => {
+        let options = downcast_into::<RawLazyCompilationOption>(self.options)?;
+        let js_backend = JsBackend::try_from(&options).unwrap();
+        plugins.push(Box::new(
+          rspack_plugin_lazy_compilation::plugin::LazyCompilationPlugin::new(
+            js_backend,
+            options.test.map(|s| {
+              RspackRegex::with_flags(&s.source, &s.flags).unwrap_or_else(|_| {
+                let msg = format!("[lazyCompilation]incorrect regex {:?}", s);
+                panic!("{msg}");
+              })
+            }),
+            options.entries,
+            options.imports,
+          ),
+        ) as Box<dyn Plugin>)
+      }
     }
     Ok(())
   }

crates/rspack_binding_options/src/options/raw_builtins/raw_lazy_compilation.rs

@@ -0,0 +1,89 @@
+use napi::{Env, JsFunction, Result};
+use napi_derive::napi;
+use rspack_binding_macros::js_fn_into_threadsafe_fn;
+use rspack_core::ModuleIdentifier;
+use rspack_napi_shared::{
+  get_napi_env,
+  threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode},
+};
+use rspack_plugin_lazy_compilation::backend::{Backend, ModuleInfo};
+
+use crate::RawRegexMatcher;
+
+#[napi(object)]
+pub struct RawModuleInfo {
+  pub active: bool,
+  pub client: String,
+  pub data: String,
+}
+
+#[napi(object)]
+pub struct RawLazyCompilationOption {
+  pub module: JsFunction,
+  pub dispose: JsFunction,
+  pub test: Option<RawRegexMatcher>,
+  pub entries: bool,
+  pub imports: bool,
+}
+
+#[napi(object)]
+pub struct RawModuleArg {
+  pub module: String,
+  pub path: String,
+}
+
+pub(crate) struct JsBackend {
+  module: ThreadsafeFunction<RawModuleArg, RawModuleInfo>,
+  dispose: ThreadsafeFunction<(), ()>,
+}
+
+impl std::fmt::Debug for JsBackend {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    f.debug_struct("JsBackend").finish()
+  }
+}
+
+impl TryFrom<&RawLazyCompilationOption> for JsBackend {
+  type Error = napi::Error;
+  fn try_from(value: &RawLazyCompilationOption) -> Result<Self> {
+    Ok(Self {
+      module: js_fn_into_threadsafe_fn!(value.module, Env::from(get_napi_env())),
+      dispose: js_fn_into_threadsafe_fn!(value.dispose, Env::from(get_napi_env())),
+    })
+  }
+}
+
+#[async_trait::async_trait]
+impl Backend for JsBackend {
+  async fn module(&mut self, identifier: ModuleIdentifier, path: String) -> ModuleInfo {
+    let module_info = self
+      .module
+      .call(
+        RawModuleArg {
+          module: identifier.to_string(),
+          path,
+        },
+        ThreadsafeFunctionCallMode::NonBlocking,
+      )
+      .unwrap()
+      .await
+      .unwrap()
+      .unwrap();
+
+    ModuleInfo {
+      active: module_info.active,
+      client: module_info.client,
+      data: module_info.data,
+    }
+  }
+
+  async fn dispose(&mut self) {
+    self
+      .dispose
+      .call((), ThreadsafeFunctionCallMode::NonBlocking)
+      .unwrap()
+      .await
+      .unwrap()
+      .unwrap();
+  }
+}

crates/rspack_core/src/compiler/compilation.rs

@@ -802,6 +802,8 @@ impl Compilation {
     //   debug_all_exports_info!(&self.module_graph);
     // }
     let start = logger.time("create chunks");
+    dbg!(self.module_graph.modules().len());
+
     use_code_splitting_cache(self, |compilation| async {
       build_chunk_graph(compilation)?;
       plugin_driver.optimize_modules(compilation).await?;

crates/rspack_core/src/compiler/hmr.rs

@@ -10,7 +10,8 @@ use rustc_hash::FxHashSet as HashSet;
 
 use super::MakeParam;
 use crate::{
-  fast_set, get_chunk_from_ukey, ChunkKind, Compilation, Compiler, ModuleGraph, RuntimeSpec,
+  fast_set, get_chunk_from_ukey, ChunkKind, Compilation, Compiler, ModuleGraph, ModuleIdentifier,
+  RuntimeSpec,
 };
 
 impl<T> Compiler<T>
@@ -21,6 +22,7 @@ where
     &mut self,
     changed_files: std::collections::HashSet<String>,
     removed_files: std::collections::HashSet<String>,
+    modified_modules: std::collections::HashSet<String>,
   ) -> Result<()> {
     let old = self.compilation.get_stats();
     let old_hash = self.compilation.hash.clone();
@@ -58,9 +60,20 @@ where
       modified_files.extend(changed_files.iter().map(PathBuf::from));
       let mut deleted_files = HashSet::default();
       deleted_files.extend(removed_files.iter().map(PathBuf::from));
+      let mut force_build_modules: HashSet<ModuleIdentifier> = HashSet::default();
+      force_build_modules.extend(
+        modified_modules
+          .iter()
+          .map(|m| ModuleIdentifier::from(m.as_str())),
+      );
 
       let mut all_files = modified_files.clone();
       all_files.extend(deleted_files.clone());
+      all_files.extend(
+        force_build_modules
+          .iter()
+          .map(|m| PathBuf::from(m.as_str())),
+      );
 
       self.cache.end_idle();
       self
@@ -129,6 +142,7 @@ where
         vec![
           MakeParam::ModifiedFiles(modified_files),
           MakeParam::DeletedFiles(deleted_files),
+          MakeParam::ForceBuildModules(force_build_modules),
         ]
       } else {
         vec![MakeParam::ForceBuildDeps(Default::default())]

crates/rspack_core/src/dependency/dependency_type.rs

@@ -93,6 +93,7 @@ pub enum DependencyType {
   /// Webpack is included
   WebpackIsIncluded,
   LoaderImport,
+  LazyImport,
   Custom(Box<str>), // TODO it will increase large layout size
 }
 
@@ -149,6 +150,7 @@ impl DependencyType {
       DependencyType::ProvideModuleForShared => Cow::Borrowed("provide module for shared"),
       DependencyType::ConsumeSharedFallback => Cow::Borrowed("consume shared fallback"),
       DependencyType::WebpackIsIncluded => Cow::Borrowed("__webpack_is_included__"),
+      DependencyType::LazyImport => Cow::Borrowed("lazy import()"),
     }
   }
 }

crates/rspack_core/src/module_factory.rs

@@ -6,7 +6,7 @@ use sugar_path::SugarPathBuf;
 
 use crate::{BoxDependency, BoxModule, Context, FactoryMeta, ModuleIdentifier, Resolve};
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct ModuleFactoryCreateData {
   pub resolve_options: Option<Box<Resolve>>,
   pub context: Context,

crates/rspack_core/src/module_graph/mod.rs

@@ -662,6 +662,7 @@ impl ModuleGraph {
     module_identifier: &ModuleIdentifier,
   ) -> (Vec<ModuleIdentifier>, &[AsyncDependenciesBlockIdentifier]) {
     let Some(m) = self.module_by_identifier(module_identifier) else {
+      dbg!(self.modules().keys());
       unreachable!("cannot find the module correspanding to {module_identifier}");
     };
     let mut deps = m