From 991cb9043f105465204b56c6ee1cc80668146cdd Mon Sep 17 00:00:00 2001 From: jserfeng <1114550440@qq.com> Date: Sat, 25 Nov 2023 21:05:04 +0800 Subject: [PATCH] feat(compilation): add queue handlers representing task_queue --- Cargo.lock | 2 + crates/node_binding/src/plugins/mod.rs | 7 +- .../rspack_core/src/compiler/compilation.rs | 78 ++++++++++++++++++- crates/rspack_core/src/compiler/queue.rs | 9 +-- .../src/dependency/dependency_type.rs | 3 + .../dependency/loader_import_dependency.rs | 49 ++++++++++++ .../rspack_core/src/normal_module_factory.rs | 1 + crates/rspack_core/src/plugin/api.rs | 5 +- crates/rspack_core/src/plugin/args.rs | 13 +++- .../rspack_core/src/plugin/plugin_driver.rs | 6 +- crates/rspack_error/src/lib.rs | 4 + crates/rspack_util/Cargo.toml | 2 + crates/rspack_util/src/fx_dashmap.rs | 7 ++ crates/rspack_util/src/lib.rs | 1 + 14 files changed, 169 insertions(+), 18 deletions(-) create mode 100644 crates/rspack_core/src/dependency/loader_import_dependency.rs create mode 100644 crates/rspack_util/src/fx_dashmap.rs diff --git a/Cargo.lock b/Cargo.lock index 248d89fab5b0..0eab36db291a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3318,8 +3318,10 @@ name = "rspack_util" version = "0.1.0" dependencies = [ "concat-string", + "dashmap", "once_cell", "regex", + "rustc-hash", "sugar_path", ] diff --git a/crates/node_binding/src/plugins/mod.rs b/crates/node_binding/src/plugins/mod.rs index 7e4b91c3ab7b..5a605ba70754 100644 --- a/crates/node_binding/src/plugins/mod.rs +++ b/crates/node_binding/src/plugins/mod.rs @@ -9,7 +9,9 @@ use rspack_binding_values::JsExecuteModuleArg; use rspack_binding_values::{AfterResolveData, JsChunkAssetArgs, JsModule}; use rspack_binding_values::{BeforeResolveData, JsAssetEmittedArgs, ToJsModule}; use rspack_binding_values::{JsResolveForSchemeInput, JsResolveForSchemeResult}; -use rspack_core::{ChunkAssetArgs, ModuleIdentifier, NormalModuleAfterResolveArgs}; +use rspack_core::{ + ChunkAssetArgs, ModuleIdentifier, NormalModuleAfterResolveArgs, SucceedModuleArgs, +}; use rspack_core::{NormalModuleBeforeResolveArgs, PluginNormalModuleFactoryAfterResolveOutput}; use rspack_core::{PluginNormalModuleFactoryBeforeResolveOutput, ResourceData}; use rspack_core::{PluginNormalModuleFactoryResolveForSchemeOutput, PluginShouldEmitHookOutput}; @@ -733,11 +735,12 @@ impl rspack_core::Plugin for JsHooksAdapter { .map_err(|err| internal_error!("Failed to call after emit: {err}"))? } - async fn succeed_module(&self, args: &dyn rspack_core::Module) -> rspack_error::Result<()> { + async fn succeed_module(&self, args: &SucceedModuleArgs<'_>) -> rspack_error::Result<()> { if self.is_hook_disabled(&Hook::SucceedModule) { return Ok(()); } let js_module = args + .module .to_js_module() .expect("Failed to convert module to JsModule"); self diff --git a/crates/rspack_core/src/compiler/compilation.rs b/crates/rspack_core/src/compiler/compilation.rs index 6a3793e4c8c6..b81fb1745902 100644 --- a/crates/rspack_core/src/compiler/compilation.rs +++ b/crates/rspack_core/src/compiler/compilation.rs @@ -23,7 +23,9 @@ use rspack_identifier::{Identifiable, IdentifierMap, IdentifierSet}; use rspack_sources::{BoxSource, CachedSource, SourceExt}; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet, FxHasher}; use swc_core::ecma::ast::ModuleItem; -use tokio::sync::mpsc::error::TryRecvError; +use tokio::sync::mpsc::{ + error::TryRecvError, unbounded_channel, UnboundedReceiver, UnboundedSender, +}; use tracing::instrument; use super::{ @@ -45,8 +47,8 @@ use crate::{ FactorizeTask, FactorizeTaskResult, Filename, Logger, Module, ModuleGraph, ModuleIdentifier, ModuleProfile, ModuleType, PathData, ProcessAssetsArgs, ProcessDependenciesQueue, ProcessDependenciesResult, ProcessDependenciesTask, RenderManifestArgs, Resolve, ResolverFactory, - RuntimeGlobals, RuntimeModule, RuntimeSpec, SharedPluginDriver, SourceType, Stats, TaskResult, - WorkerTask, + RuntimeGlobals, RuntimeModule, RuntimeSpec, SharedPluginDriver, SourceType, Stats, + SucceedModuleArgs, TaskResult, WorkerTask, }; use crate::{tree_shaking::visitor::OptimizeAnalyzeResult, Context}; @@ -110,6 +112,11 @@ pub struct Compilation { pub build_dependencies: IndexSet>, pub side_effects_free_modules: IdentifierSet, pub module_item_map: IdentifierMap>, + + pub factorize_queue: QueueHandler, + pub add_queue: QueueHandler, + pub build_queue: QueueHandler, + pub process_dependencies_queue: QueueHandler, } impl Compilation { @@ -170,6 +177,11 @@ impl Compilation { side_effects_free_modules: IdentifierSet::default(), module_item_map: IdentifierMap::default(), include_module_ids: IdentifierSet::default(), + + factorize_queue: QueueHandler::new(), + add_queue: QueueHandler::new(), + build_queue: QueueHandler::new(), + process_dependencies_queue: QueueHandler::new(), } } @@ -431,10 +443,12 @@ impl Compilation { let mut active_task_count = 0usize; let is_expected_shutdown = Arc::new(AtomicBool::new(false)); let (result_tx, mut result_rx) = tokio::sync::mpsc::unbounded_channel::>(); + let mut factorize_queue = FactorizeQueue::new(); let mut add_queue = AddQueue::new(); let mut build_queue = BuildQueue::new(); let mut process_dependencies_queue = ProcessDependenciesQueue::new(); + let mut make_failed_dependencies: HashSet = HashSet::default(); let mut make_failed_module = HashSet::default(); let mut errored = None; @@ -489,6 +503,19 @@ impl Compilation { } tokio::task::block_in_place(|| loop { + if let Ok(task) = self.factorize_queue.try_recv() { + factorize_queue.add_task(task); + } + if let Ok(task) = self.add_queue.try_recv() { + add_queue.add_task(task); + } + if let Ok(task) = self.build_queue.try_recv() { + build_queue.add_task(task); + } + if let Ok(task) = self.process_dependencies_queue.try_recv() { + process_dependencies_queue.add_task(task); + } + let start = factorize_time.start(); while let Some(task) = factorize_queue.get_task() { tokio::spawn({ @@ -724,6 +751,18 @@ impl Compilation { from_cache, } = task_result; + tokio::runtime::Handle::current().block_on(async { + self + .plugin_driver + .succeed_module(&SucceedModuleArgs { + module: &module, + build_result: &build_result, + compilation: self, + }) + .await + .unwrap_or_else(|e| panic!("Run succeed_module hook failed: {}", e)); + }); + if let Some(counter) = &mut build_cache_counter { if from_cache { counter.hit(); @@ -1844,3 +1883,36 @@ impl AssetInfo { pub struct AssetInfoRelated { pub source_map: Option, } + +#[derive(Debug)] +pub struct QueueHandler(UnboundedSender, UnboundedReceiver); + +impl Default for QueueHandler { + fn default() -> Self { + let (tx, rx) = unbounded_channel(); + Self(tx, rx) + } +} + +impl QueueHandler { + pub fn new() -> Self { + Self::default() + } + + pub fn add(&self, task: T) -> Result<()> { + self.0.send(task).map_err(|reason| { + rspack_error::Error::InternalError(rspack_error::InternalError::new( + format!("Failed to add queue task, reason: {}", reason), + Severity::Error, + )) + }) + } + + pub fn get_tx(&self) -> UnboundedSender { + self.0.clone() + } + + pub fn try_recv(&mut self) -> std::result::Result { + self.1.try_recv() + } +} diff --git a/crates/rspack_core/src/compiler/queue.rs b/crates/rspack_core/src/compiler/queue.rs index 9c45c8a6f392..d4bacc846e6a 100644 --- a/crates/rspack_core/src/compiler/queue.rs +++ b/crates/rspack_core/src/compiler/queue.rs @@ -24,6 +24,7 @@ pub trait WorkerTask { async fn run(self) -> Result; } +#[derive(Debug)] pub struct FactorizeTask { pub original_module_identifier: Option, pub original_module_context: Option>, @@ -170,6 +171,7 @@ impl WorkerTask for FactorizeTask { pub type FactorizeQueue = WorkerQueue; +#[derive(Debug)] pub struct AddTask { pub original_module_identifier: Option, pub module: Box, @@ -256,6 +258,7 @@ fn set_resolved_module( pub type AddQueue = WorkerQueue; +#[derive(Debug)] pub struct BuildTask { pub module: Box, pub resolver_factory: Arc, @@ -308,11 +311,6 @@ impl WorkerTask for BuildTask { }) .await; - plugin_driver - .succeed_module(&**module) - .await - .unwrap_or_else(|e| panic!("Run succeed_module hook failed: {}", e)); - result.map(|t| (t, module)) }) .await @@ -345,6 +343,7 @@ impl WorkerTask for BuildTask { pub type BuildQueue = WorkerQueue; +#[derive(Debug)] pub struct ProcessDependenciesTask { pub original_module_identifier: ModuleIdentifier, pub dependencies: Vec, diff --git a/crates/rspack_core/src/dependency/dependency_type.rs b/crates/rspack_core/src/dependency/dependency_type.rs index 21115f520e3f..8782ad6ba6fb 100644 --- a/crates/rspack_core/src/dependency/dependency_type.rs +++ b/crates/rspack_core/src/dependency/dependency_type.rs @@ -58,6 +58,8 @@ pub enum DependencyType { RequireResolve, /// wasm import WasmImport, + /// import module + LoaderImport, /// wasm export import WasmExportImported, /// static exports @@ -99,6 +101,7 @@ impl DependencyType { DependencyType::RequireContext => Cow::Borrowed("require.context"), DependencyType::RequireResolve => Cow::Borrowed("require.resolve"), DependencyType::WasmImport => Cow::Borrowed("wasm import"), + DependencyType::LoaderImport => Cow::Borrowed("loader import"), DependencyType::WasmExportImported => Cow::Borrowed("wasm export imported"), DependencyType::StaticExports => Cow::Borrowed("static exports"), DependencyType::Custom(ty) => Cow::Owned(format!("custom {ty}")), diff --git a/crates/rspack_core/src/dependency/loader_import_dependency.rs b/crates/rspack_core/src/dependency/loader_import_dependency.rs new file mode 100644 index 000000000000..49d200b8494a --- /dev/null +++ b/crates/rspack_core/src/dependency/loader_import_dependency.rs @@ -0,0 +1,49 @@ +use crate::{ + AsContextDependency, AsDependencyTemplate, Dependency, DependencyId, ModuleDependency, +}; + +#[derive(Debug, Hash, PartialEq, Eq, Clone)] +pub struct LoaderImportDependency { + request: String, + id: DependencyId, +} + +impl LoaderImportDependency { + pub fn new(request: String) -> Self { + Self { + request, + id: DependencyId::new(), + } + } +} + +impl AsDependencyTemplate for LoaderImportDependency {} +impl AsContextDependency for LoaderImportDependency {} + +impl Dependency for LoaderImportDependency { + fn dependency_debug_name(&self) -> &'static str { + "LoaderImportDependency" + } + + fn id(&self) -> &crate::DependencyId { + &self.id + } + + fn dependency_type(&self) -> &crate::DependencyType { + &crate::DependencyType::LoaderImport + } +} + +impl ModuleDependency for LoaderImportDependency { + fn request(&self) -> &str { + &self.request + } + + fn user_request(&self) -> &str { + &self.request + } + + fn set_request(&mut self, request: String) { + self.request = request + } +} diff --git a/crates/rspack_core/src/normal_module_factory.rs b/crates/rspack_core/src/normal_module_factory.rs index 2121ea94223f..a173a973e022 100644 --- a/crates/rspack_core/src/normal_module_factory.rs +++ b/crates/rspack_core/src/normal_module_factory.rs @@ -636,6 +636,7 @@ impl NormalModuleFactory { .plugin_driver .create_module(NormalModuleCreateData { dependency_type: data.dependency.dependency_type().clone(), + dependency_id: *data.dependency.id(), }) .await? { diff --git a/crates/rspack_core/src/plugin/api.rs b/crates/rspack_core/src/plugin/api.rs index 629be930a149..f8a35d69c045 100644 --- a/crates/rspack_core/src/plugin/api.rs +++ b/crates/rspack_core/src/plugin/api.rs @@ -14,7 +14,8 @@ use crate::{ NormalModule, NormalModuleAfterResolveArgs, NormalModuleBeforeResolveArgs, NormalModuleCreateData, NormalModuleFactoryContext, OptimizeChunksArgs, ParserAndGenerator, PluginContext, ProcessAssetsArgs, RenderArgs, RenderChunkArgs, RenderManifestArgs, - RenderModuleContentArgs, RenderStartupArgs, Resolver, SourceType, ThisCompilationArgs, + RenderModuleContentArgs, RenderStartupArgs, Resolver, SourceType, SucceedModuleArgs, + ThisCompilationArgs, }; // use anyhow::{Context, Result}; @@ -435,7 +436,7 @@ pub trait Plugin: Debug + Send + Sync { Ok(()) } - async fn succeed_module(&self, _module: &dyn Module) -> Result<()> { + async fn succeed_module(&self, _args: &SucceedModuleArgs<'_>) -> Result<()> { Ok(()) } diff --git a/crates/rspack_core/src/plugin/args.rs b/crates/rspack_core/src/plugin/args.rs index 559a103dbe59..fa9b2833872d 100644 --- a/crates/rspack_core/src/plugin/args.rs +++ b/crates/rspack_core/src/plugin/args.rs @@ -6,9 +6,9 @@ use rspack_sources::BoxSource; use rustc_hash::FxHashSet as HashSet; use crate::{ - Chunk, ChunkInitFragments, ChunkUkey, Compilation, Context, DependencyCategory, DependencyType, - ErrorSpan, FactoryMeta, ModuleDependency, ModuleGraphModule, ModuleIdentifier, Resolve, - RuntimeGlobals, SharedPluginDriver, Stats, + BoxModule, BuildResult, Chunk, ChunkInitFragments, ChunkUkey, Compilation, Context, + DependencyCategory, DependencyId, DependencyType, ErrorSpan, FactoryMeta, ModuleDependency, + ModuleGraphModule, ModuleIdentifier, Resolve, RuntimeGlobals, SharedPluginDriver, Stats, }; #[derive(Debug)] @@ -84,6 +84,7 @@ pub struct FactorizeArgs<'me> { #[derive(Debug, Clone)] pub struct NormalModuleCreateData { pub dependency_type: DependencyType, + pub dependency_id: DependencyId, } #[derive(Debug, Clone)] @@ -247,3 +248,9 @@ impl<'me> JsChunkHashArgs<'me> { .expect("chunk should exist in chunk_by_ukey") } } + +pub struct SucceedModuleArgs<'me> { + pub module: &'me BoxModule, + pub build_result: &'me BuildResult, + pub compilation: &'me Compilation, +} diff --git a/crates/rspack_core/src/plugin/plugin_driver.rs b/crates/rspack_core/src/plugin/plugin_driver.rs index b4f9a7528bc0..725754a74f4c 100644 --- a/crates/rspack_core/src/plugin/plugin_driver.rs +++ b/crates/rspack_core/src/plugin/plugin_driver.rs @@ -24,7 +24,7 @@ use crate::{ PluginRenderManifestHookOutput, PluginRenderModuleContentOutput, PluginRenderStartupHookOutput, PluginThisCompilationHookOutput, ProcessAssetsArgs, RenderArgs, RenderChunkArgs, RenderManifestArgs, RenderModuleContentArgs, RenderStartupArgs, Resolver, ResolverFactory, Stats, - ThisCompilationArgs, + SucceedModuleArgs, ThisCompilationArgs, }; pub struct PluginDriver { @@ -589,9 +589,9 @@ impl PluginDriver { } #[instrument(name = "plugin:succeed_module", skip_all)] - pub async fn succeed_module(&self, module: &dyn Module) -> Result<()> { + pub async fn succeed_module(&self, args: &SucceedModuleArgs<'_>) -> Result<()> { for plugin in &self.plugins { - plugin.succeed_module(module).await?; + plugin.succeed_module(args).await?; } Ok(()) } diff --git a/crates/rspack_error/src/lib.rs b/crates/rspack_error/src/lib.rs index 7d90fc7bc452..e0c8b2f48a4d 100644 --- a/crates/rspack_error/src/lib.rs +++ b/crates/rspack_error/src/lib.rs @@ -37,6 +37,10 @@ impl TWithDiagnosticArray { pub fn split_into_parts(self) -> (T, Vec) { (self.inner, self.diagnostic) } + + pub fn get(&self) -> &T { + &self.inner + } } impl Clone for TWithDiagnosticArray { diff --git a/crates/rspack_util/Cargo.toml b/crates/rspack_util/Cargo.toml index f866145e2bc3..6c55c5d6620a 100644 --- a/crates/rspack_util/Cargo.toml +++ b/crates/rspack_util/Cargo.toml @@ -9,6 +9,8 @@ version = "0.1.0" [dependencies] concat-string = { workspace = true } +dashmap = { workspace = true } once_cell = { workspace = true } regex = { workspace = true } +rustc-hash = { workspace = true } sugar_path = { workspace = true } diff --git a/crates/rspack_util/src/fx_dashmap.rs b/crates/rspack_util/src/fx_dashmap.rs new file mode 100644 index 000000000000..d778e5e0a05a --- /dev/null +++ b/crates/rspack_util/src/fx_dashmap.rs @@ -0,0 +1,7 @@ +use std::hash::BuildHasherDefault; + +use dashmap::{DashMap, DashSet}; +use rustc_hash::FxHasher; + +pub type FxDashMap = DashMap>; +pub type FxDashSet = DashSet>; diff --git a/crates/rspack_util/src/lib.rs b/crates/rspack_util/src/lib.rs index fd252cce4eda..76cf7932513e 100644 --- a/crates/rspack_util/src/lib.rs +++ b/crates/rspack_util/src/lib.rs @@ -3,6 +3,7 @@ use std::future::Future; pub mod comparators; pub mod ext; +pub mod fx_dashmap; pub mod identifier; pub mod number_hash; pub mod swc;