From 4795cf01f9f2031223c57972454353fc33a3b7c4 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:44:33 +0100 Subject: [PATCH 01/11] Migrate global module ids to single-graph --- Cargo.lock | 1 + crates/next-api/Cargo.toml | 1 + crates/next-api/src/app.rs | 72 +++++-- .../next-api/src/global_module_id_strategy.rs | 90 -------- crates/next-api/src/lib.rs | 1 - crates/next-api/src/middleware.rs | 24 ++- crates/next-api/src/module_graph.rs | 194 +++++++++++++----- crates/next-api/src/project.rs | 101 +++++---- crates/next-api/src/route.rs | 11 +- crates/next-api/src/server_actions.rs | 16 +- .../next_app/app_client_references_chunks.rs | 19 +- crates/next-core/src/next_config.rs | 6 +- .../server_component_module.rs | 27 +-- test/e2e/app-dir/build-size/index.test.ts | 7 +- .../non-root-project-monorepo.test.ts | 2 +- .../turbopack-browser/src/chunking_context.rs | 7 +- .../src/chunk/chunking_context.rs | 9 + .../src/chunk/module_id_strategies.rs | 29 ++- turbopack/crates/turbopack-core/src/module.rs | 4 - .../turbopack-core/src/module_graph/mod.rs | 112 ++++++++-- .../src/async_chunk/module.rs | 4 +- .../src/global_module_id_strategy.rs | 177 +--------------- .../src/references/external_module.rs | 2 +- .../src/tree_shake/chunk_item.rs | 21 +- .../turbopack-nodejs/src/chunking_context.rs | 13 +- 25 files changed, 480 insertions(+), 470 deletions(-) delete mode 100644 crates/next-api/src/global_module_id_strategy.rs diff --git a/Cargo.lock b/Cargo.lock index 9050ea4f39a9e..f7b3799fa98e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4324,6 +4324,7 @@ dependencies = [ "turbo-tasks-build", "turbo-tasks-env", "turbo-tasks-fs", + "turbo-tasks-hash", "turbo-tasks-memory", "turbopack", "turbopack-browser", diff --git a/crates/next-api/Cargo.toml b/crates/next-api/Cargo.toml index af30185e2ce2f..9cfadce352138 100644 --- a/crates/next-api/Cargo.toml +++ b/crates/next-api/Cargo.toml @@ -28,6 +28,7 @@ turbo-rcstr = { workspace = true } turbo-tasks = { workspace = true } turbo-tasks-env = { workspace = true } turbo-tasks-fs = { workspace = true } +turbo-tasks-hash = { workspace = true } turbo-tasks-memory = { workspace = true } turbopack = { workspace = true } turbopack-browser = { workspace = true } diff --git a/crates/next-api/src/app.rs b/crates/next-api/src/app.rs index 92ca5b69cd80d..f603ebf484f30 100644 --- a/crates/next-api/src/app.rs +++ b/crates/next-api/src/app.rs @@ -11,8 +11,7 @@ use next_core::{ get_edge_resolve_options_context, get_next_package, next_app::{ get_app_client_references_chunks, get_app_client_shared_chunk_group, get_app_page_entry, - get_app_route_entry, include_modules_module::IncludeModulesModule, - metadata::route::get_app_metadata_route_entry, AppEntry, AppPage, + get_app_route_entry, metadata::route::get_app_metadata_route_entry, AppEntry, AppPage, }, next_client::{ get_client_module_options_context, get_client_resolve_options_context, @@ -52,7 +51,7 @@ use turbopack::{ use turbopack_core::{ asset::AssetContent, chunk::{ - availability_info::AvailabilityInfo, ChunkingContext, ChunkingContextExt, + availability_info::AvailabilityInfo, ChunkableModule, ChunkingContext, ChunkingContextExt, EntryChunkGroupResult, EvaluatableAsset, EvaluatableAssets, }, file_source::FileSource, @@ -72,7 +71,7 @@ use crate::{ }, font::create_font_manifest, loadable_manifest::create_react_loadable_manifest, - module_graph::get_reduced_graphs_for_endpoint, + module_graph::{get_reduced_graphs_for_endpoint, ReducedGraphs}, nft_json::NftJsonAsset, paths::{ all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root, @@ -80,7 +79,7 @@ use crate::{ }, project::Project, route::{AppPageRoute, Endpoint, Route, Routes, WrittenEndpoint}, - server_actions::create_server_actions_manifest, + server_actions::{build_server_actions_loader, create_server_actions_manifest}, webpack_stats::generate_webpack_stats, }; @@ -1522,16 +1521,23 @@ impl AppEndpoint { let client_references = client_references.await?; let span = tracing::trace_span!("server utils"); async { - let utils_module = IncludeModulesModule::new( - AssetIdent::from_path(this.app_project.project().project_path()) - .with_modifier(server_utils_modifier()), - client_references.server_utils.iter().map(|v| **v).collect(), - ); - + let server_utils = client_references + .server_utils + .iter() + .map(|m| async move { + Ok(*ResolvedVc::try_downcast::>(*m) + .await? + .context("Expected server utils to be chunkable")?) + }) + .try_join() + .await?; let chunk_group = chunking_context - .chunk_group( - utils_module.ident(), - Vc::upcast(utils_module), + .chunk_group_multiple( + AssetIdent::from_path( + this.app_project.project().project_path(), + ) + .with_modifier(server_utils_modifier()), + server_utils, Value::new(current_availability_info), ) .await?; @@ -1736,8 +1742,42 @@ impl Endpoint for AppEndpoint { #[turbo_tasks::function] async fn root_modules(self: Vc) -> Result> { - let rsc_entry = self.app_endpoint_entry().await?.rsc_entry; - Ok(Vc::cell(vec![rsc_entry])) + Ok(Vc::cell(vec![self.app_endpoint_entry().await?.rsc_entry])) + } + + #[turbo_tasks::function] + async fn additional_root_modules( + self: Vc, + graphs: Vc, + ) -> Result> { + let this = self.await?; + let app_entry = self.app_endpoint_entry().await?; + let rsc_entry = app_entry.rsc_entry; + let runtime = app_entry.config.await?.runtime.unwrap_or_default(); + + let actions = graphs.get_server_actions_for_endpoint( + *rsc_entry, + match runtime { + NextRuntime::Edge => Vc::upcast(this.app_project.edge_rsc_module_context()), + NextRuntime::NodeJs => Vc::upcast(this.app_project.rsc_module_context()), + }, + ); + + let server_actions_loader = ResolvedVc::upcast( + build_server_actions_loader( + this.app_project.project().project_path(), + app_entry.original_name.clone(), + actions, + match runtime { + NextRuntime::Edge => Vc::upcast(this.app_project.edge_rsc_module_context()), + NextRuntime::NodeJs => Vc::upcast(this.app_project.rsc_module_context()), + }, + ) + .to_resolved() + .await?, + ); + + Ok(Vc::cell(vec![server_actions_loader])) } } diff --git a/crates/next-api/src/global_module_id_strategy.rs b/crates/next-api/src/global_module_id_strategy.rs deleted file mode 100644 index 5ec6719d5a771..0000000000000 --- a/crates/next-api/src/global_module_id_strategy.rs +++ /dev/null @@ -1,90 +0,0 @@ -use anyhow::Result; -use turbo_tasks::Vc; -use turbopack_core::chunk::module_id_strategies::{GlobalModuleIdStrategy, ModuleIdStrategy}; -use turbopack_ecmascript::global_module_id_strategy::{ - children_modules_idents, merge_preprocessed_module_ids, PreprocessedChildrenIdents, -}; - -use crate::{ - project::Project, - route::{Endpoint, Route}, -}; - -#[turbo_tasks::value] -pub struct GlobalModuleIdStrategyBuilder; - -// NOTE(LichuAcu) To access all entrypoints, we need to access an instance of `Project`, but -// `Project` is not available in `turbopack-core`, so we need need this -// `GlobalModuleIdStrategyBuilder` in `next-api`. -#[turbo_tasks::value_impl] -impl GlobalModuleIdStrategyBuilder { - #[turbo_tasks::function] - pub async fn build(project: Vc) -> Result>> { - let mut preprocessed_module_ids = Vec::new(); - - preprocessed_module_ids.push(children_modules_idents(project.client_main_modules())); - - let entrypoints = project.entrypoints().await?; - - preprocessed_module_ids.push(preprocess_module_ids(*entrypoints.pages_error_endpoint)); - preprocessed_module_ids.push(preprocess_module_ids(*entrypoints.pages_app_endpoint)); - preprocessed_module_ids.push(preprocess_module_ids(*entrypoints.pages_document_endpoint)); - - if let Some(middleware) = &entrypoints.middleware { - preprocessed_module_ids.push(preprocess_module_ids(middleware.endpoint)); - } - - if let Some(instrumentation) = &entrypoints.instrumentation { - let node_js = instrumentation.node_js; - let edge = instrumentation.edge; - preprocessed_module_ids.push(preprocess_module_ids(node_js)); - preprocessed_module_ids.push(preprocess_module_ids(edge)); - } - - for (_, route) in entrypoints.routes.iter() { - match route { - Route::Page { - html_endpoint, - data_endpoint, - } => { - preprocessed_module_ids.push(preprocess_module_ids(**html_endpoint)); - preprocessed_module_ids.push(preprocess_module_ids(**data_endpoint)); - } - Route::PageApi { endpoint } => { - preprocessed_module_ids.push(preprocess_module_ids(**endpoint)); - } - Route::AppPage(page_routes) => { - for page_route in page_routes { - preprocessed_module_ids - .push(preprocess_module_ids(page_route.html_endpoint)); - preprocessed_module_ids - .push(preprocess_module_ids(page_route.rsc_endpoint)); - } - } - Route::AppRoute { - original_name: _, - endpoint, - } => { - preprocessed_module_ids.push(preprocess_module_ids(**endpoint)); - } - Route::Conflict => { - tracing::info!("WARN: conflict"); - } - } - } - - let module_id_map = merge_preprocessed_module_ids(preprocessed_module_ids).await?; - - Ok(Vc::upcast( - GlobalModuleIdStrategy::new(module_id_map).await?, - )) - } -} - -// NOTE(LichuAcu) We can't move this function to `turbopack-core` because we need access to -// `Endpoint`, which is not available there. -#[turbo_tasks::function] -fn preprocess_module_ids(endpoint: Vc>) -> Vc { - let root_modules = endpoint.root_modules(); - children_modules_idents(root_modules) -} diff --git a/crates/next-api/src/lib.rs b/crates/next-api/src/lib.rs index 26fe340398f73..c1859ddc503d2 100644 --- a/crates/next-api/src/lib.rs +++ b/crates/next-api/src/lib.rs @@ -9,7 +9,6 @@ mod dynamic_imports; mod empty; pub mod entrypoints; mod font; -pub mod global_module_id_strategy; mod instrumentation; mod loadable_manifest; mod middleware; diff --git a/crates/next-api/src/middleware.rs b/crates/next-api/src/middleware.rs index f25b6070437ae..2439c62d8b95e 100644 --- a/crates/next-api/src/middleware.rs +++ b/crates/next-api/src/middleware.rs @@ -64,7 +64,7 @@ impl MiddlewareEndpoint { } #[turbo_tasks::function] - async fn edge_files(&self) -> Result> { + async fn entry_module(&self) -> Vc> { let userland_module = self .asset_context .process( @@ -79,22 +79,28 @@ impl MiddlewareEndpoint { userland_module, ); - let module = wrap_edge_entry( + wrap_edge_entry( *self.asset_context, self.project.project_path(), module, "middleware".into(), - ); + ) + } + + #[turbo_tasks::function] + async fn edge_files(self: Vc) -> Result> { + let this = self.await?; + let module = self.entry_module(); let mut evaluatable_assets = get_server_runtime_entries( Value::new(ServerContextType::Middleware { - app_dir: self.app_dir, - ecmascript_client_reference_transition_name: self + app_dir: this.app_dir, + ecmascript_client_reference_transition_name: this .ecmascript_client_reference_transition_name, }), - self.project.next_mode(), + this.project.next_mode(), ) - .resolve_entries(*self.asset_context) + .resolve_entries(*this.asset_context) .await? .clone_value(); @@ -109,7 +115,7 @@ impl MiddlewareEndpoint { .context("Entry module must be evaluatable")?; evaluatable_assets.push(evaluatable.to_resolved().await?); - let edge_chunking_context = self.project.edge_chunking_context(false); + let edge_chunking_context = this.project.edge_chunking_context(false); let edge_files = edge_chunking_context.evaluated_chunk_group_assets( module.ident(), @@ -322,7 +328,7 @@ impl Endpoint for MiddlewareEndpoint { #[turbo_tasks::function] async fn root_modules(self: Vc) -> Result> { - Ok(Vc::cell(vec![self.userland_module().to_resolved().await?])) + Ok(Vc::cell(vec![self.entry_module().to_resolved().await?])) } } diff --git a/crates/next-api/src/module_graph.rs b/crates/next-api/src/module_graph.rs index 22dad0deaf8a7..1421e04e5ea8f 100644 --- a/crates/next-api/src/module_graph.rs +++ b/crates/next-api/src/module_graph.rs @@ -15,14 +15,21 @@ use next_core::{ }; use tracing::Instrument; use turbo_tasks::{ - CollectiblesSource, FxIndexMap, FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc, + CollectiblesSource, FxIndexMap, FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, + ValueToString, Vc, }; +use turbo_tasks_hash::hash_xxh3_hash64; use turbopack_core::{ + chunk::{module_id_strategies::GlobalModuleIdStrategy, ChunkingType}, context::AssetContext, issue::Issue, module::Module, module_graph::{GraphTraversalAction, SingleModuleGraph}, }; +use turbopack_ecmascript::{ + async_chunk::module::async_loader_modifier, + global_module_id_strategy::merge_preprocessed_module_ids, +}; use crate::{ client_references::{map_client_references, ClientReferenceMapType, ClientReferencesSet}, @@ -48,7 +55,7 @@ async fn get_module_graph_for_endpoint( let mut graphs = vec![]; let mut visited_modules = if !server_utils.is_empty() { - let graph = SingleModuleGraph::new_with_entries_visited( + let graph = SingleModuleGraph::new_with_entries_visited_root( *entry, server_utils.iter().map(|m| **m).collect(), Vc::cell(Default::default()), @@ -67,7 +74,7 @@ async fn get_module_graph_for_endpoint( // ast-grep-ignore: to-resolved-in-loop for module in server_component_entries.iter() { - let graph = SingleModuleGraph::new_with_entries_visited( + let graph = SingleModuleGraph::new_with_entries_visited_root( *entry, vec![Vc::upcast(**module)], Vc::cell(visited_modules.clone()), @@ -85,7 +92,7 @@ async fn get_module_graph_for_endpoint( // Any previous iteration above would have added the entry node, but not actually visited it. visited_modules.remove(&entry); - let graph = SingleModuleGraph::new_with_entries_visited( + let graph = SingleModuleGraph::new_with_entries_visited_root( *entry, vec![*entry], Vc::cell(visited_modules.clone()), @@ -97,6 +104,33 @@ async fn get_module_graph_for_endpoint( Ok(Vc::cell(graphs)) } +#[turbo_tasks::function] +async fn get_module_graph_for_project(project: ResolvedVc) -> Vc { + SingleModuleGraph::new_with_entries(project.get_all_entries()) +} + +#[turbo_tasks::function] +async fn get_additional_module_graph_for_project( + project: ResolvedVc, + graph: Vc, +) -> Result> { + let visited_modules: HashSet<_> = graph.await?.iter_nodes().map(|n| n.module).collect(); + let entries = project.get_all_entries().await?; + let additional_entries = project + .get_all_additional_entries(ReducedGraphs::new(vec![graph], false)) + .await?; + let collect = entries + .iter() + .copied() + .chain(additional_entries.iter().copied()) + .collect(); + + Ok(SingleModuleGraph::new_with_entries_visited( + Vc::cell(collect), + Vc::cell(visited_modules), + )) +} + #[turbo_tasks::value] pub struct NextDynamicGraph { is_single_page: bool, @@ -459,6 +493,52 @@ pub struct ReducedGraphs { #[turbo_tasks::value_impl] impl ReducedGraphs { + #[turbo_tasks::function] + async fn new(graphs: Vec>, is_single_page: bool) -> Result> { + let next_dynamic = async { + graphs + .iter() + .map(|graph| { + NextDynamicGraph::new_with_entries(*graph, is_single_page).to_resolved() + }) + .try_join() + .await + } + .instrument(tracing::info_span!("generating next/dynamic graphs")) + .await?; + + let server_actions = async { + graphs + .iter() + .map(|graph| { + ServerActionsGraph::new_with_entries(*graph, is_single_page).to_resolved() + }) + .try_join() + .await + } + .instrument(tracing::info_span!("generating server actions graphs")) + .await?; + + let client_references = async { + graphs + .iter() + .map(|graph| { + ClientReferencesGraph::new_with_entries(*graph, is_single_page).to_resolved() + }) + .try_join() + .await + } + .instrument(tracing::info_span!("generating client references graphs")) + .await?; + + Ok(Self { + next_dynamic, + server_actions, + client_references, + } + .cell()) + } + /// Returns the next/dynamic-ally imported (client) modules (from RSC and SSR modules) for the /// given endpoint. #[turbo_tasks::function] @@ -593,13 +673,17 @@ async fn get_reduced_graphs_for_endpoint_inner_operation( async move { get_module_graph_for_endpoint(*entry).await } .instrument(tracing::info_span!("module graph for endpoint")) .await? - .clone_value(), + .iter() + .map(|v| **v) + .collect(), ), NextMode::Build => ( false, vec![ - async move { - get_global_module_graph(*project) + *async move { + get_module_graph_for_project(*project) + // This is a performance optimization. This function is a root aggregation + // function that aggregates over the whole subgraph. .resolve_strongly_consistent() .await? .to_resolved() @@ -611,46 +695,7 @@ async fn get_reduced_graphs_for_endpoint_inner_operation( ), }; - let next_dynamic = async { - graphs - .iter() - .map(|graph| NextDynamicGraph::new_with_entries(**graph, is_single_page).to_resolved()) - .try_join() - .await - } - .instrument(tracing::info_span!("generating next/dynamic graphs")) - .await?; - - let server_actions = async { - graphs - .iter() - .map(|graph| { - ServerActionsGraph::new_with_entries(**graph, is_single_page).to_resolved() - }) - .try_join() - .await - } - .instrument(tracing::info_span!("generating server actions graphs")) - .await?; - - let client_references = async { - graphs - .iter() - .map(|graph| { - ClientReferencesGraph::new_with_entries(**graph, is_single_page).to_resolved() - }) - .try_join() - .await - } - .instrument(tracing::info_span!("generating client references graphs")) - .await?; - - Ok(ReducedGraphs { - next_dynamic, - server_actions, - client_references, - } - .cell()) + Ok(ReducedGraphs::new(graphs, is_single_page)) } /// Generates a [ReducedGraph] for the given project and endpoint containing information that is @@ -671,3 +716,58 @@ pub async fn get_reduced_graphs_for_endpoint( } Ok(result_vc) } + +#[turbo_tasks::function] +pub async fn get_global_module_id_strategy( + project: Vc, +) -> Result> { + let graph_op = get_module_graph_for_project(project); + // TODO get rid of this once everything inside calls `take_collectibles()` when needed + let graph = graph_op.strongly_consistent().await?; + let _ = graph_op.take_collectibles::>(); + + let additional_graph_op = get_additional_module_graph_for_project(project, graph_op); + // TODO get rid of this once everything inside calls `take_collectibles()` when needed + let additional_graph = additional_graph_op.strongly_consistent().await?; + let _ = additional_graph_op.take_collectibles::>(); + + let graphs = [graph, additional_graph]; + + let mut idents: Vec<_> = graphs + .iter() + .flat_map(|graph| graph.iter_nodes()) + .map(|node| node.module.ident()) + .collect(); + + for graph in graphs.iter() { + // Additionally, add all the modules that are inserted by chunking (i.e. async loaders) + graph.traverse_edges(|(parent, current)| { + if let Some((_, &ChunkingType::Async)) = parent { + idents.push( + current + .module + .ident() + .with_modifier(async_loader_modifier()), + ); + } + GraphTraversalAction::Continue + })?; + } + + let module_id_map = idents + .into_iter() + .map(|ident| ident.to_string()) + .try_join() + .await? + .iter() + .map(|module_ident| { + let ident_str = module_ident.clone_value(); + let hash = hash_xxh3_hash64(&ident_str); + (ident_str, hash) + }) + .collect(); + + let module_id_map = merge_preprocessed_module_ids(&module_id_map).await?; + + GlobalModuleIdStrategy::new(module_id_map).await +} diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs index 159f7a5dc36d7..a7ad81751ebe5 100644 --- a/crates/next-api/src/project.rs +++ b/crates/next-api/src/project.rs @@ -64,11 +64,11 @@ use crate::{ build, empty::EmptyEndpoint, entrypoints::Entrypoints, - global_module_id_strategy::GlobalModuleIdStrategyBuilder, instrumentation::InstrumentationEndpoint, middleware::MiddlewareEndpoint, + module_graph::{get_global_module_id_strategy, ReducedGraphs}, pages::PagesProject, - route::{AppPageRoute, Endpoint, Route}, + route::{AppPageRoute, Endpoint, Endpoints, Route}, versioned_content_map::VersionedContentMap, }; @@ -735,36 +735,22 @@ impl Project { } #[turbo_tasks::function] - pub async fn get_all_entries(self: Vc) -> Result> { - let mut modules = Vec::new(); - - async fn add_endpoint( - endpoint: Vc>, - modules: &mut Vec>>, - ) -> Result<()> { - let root_modules = endpoint.root_modules().await?; - modules.extend(root_modules.iter().copied()); - Ok(()) - } - - modules.extend(self.client_main_modules().await?.iter().copied()); + pub async fn get_all_endpoints(self: Vc) -> Result> { + let mut endpoints = vec![]; let entrypoints = self.entrypoints().await?; - modules.extend(self.client_main_modules().await?.iter().copied()); - add_endpoint(*entrypoints.pages_error_endpoint, &mut modules).await?; - add_endpoint(*entrypoints.pages_app_endpoint, &mut modules).await?; - add_endpoint(*entrypoints.pages_document_endpoint, &mut modules).await?; + endpoints.push(entrypoints.pages_error_endpoint); + endpoints.push(entrypoints.pages_app_endpoint); + endpoints.push(entrypoints.pages_document_endpoint); if let Some(middleware) = &entrypoints.middleware { - add_endpoint(middleware.endpoint, &mut modules).await?; + endpoints.push(middleware.endpoint.to_resolved().await?); } if let Some(instrumentation) = &entrypoints.instrumentation { - let node_js = instrumentation.node_js; - let edge = instrumentation.edge; - add_endpoint(node_js, &mut modules).await?; - add_endpoint(edge, &mut modules).await?; + endpoints.push(instrumentation.node_js.to_resolved().await?); + endpoints.push(instrumentation.edge.to_resolved().await?); } for (_, route) in entrypoints.routes.iter() { @@ -773,10 +759,10 @@ impl Project { html_endpoint, data_endpoint: _, } => { - add_endpoint(**html_endpoint, &mut modules).await?; + endpoints.push(*html_endpoint); } Route::PageApi { endpoint } => { - add_endpoint(**endpoint, &mut modules).await?; + endpoints.push(*endpoint); } Route::AppPage(page_routes) => { for AppPageRoute { @@ -785,14 +771,14 @@ impl Project { rsc_endpoint: _, } in page_routes { - add_endpoint(*html_endpoint, &mut modules).await?; + endpoints.push(html_endpoint.to_resolved().await?); } } Route::AppRoute { original_name: _, endpoint, } => { - add_endpoint(**endpoint, &mut modules).await?; + endpoints.push(*endpoint); } Route::Conflict => { tracing::info!("WARN: conflict"); @@ -800,6 +786,41 @@ impl Project { } } + Ok(Vc::cell(endpoints)) + } + + #[turbo_tasks::function] + pub async fn get_all_entries(self: Vc) -> Result> { + let mut modules: Vec>> = self + .get_all_endpoints() + .await? + .iter() + .map(|endpoint| endpoint.root_modules()) + .try_flat_join() + .await? + .into_iter() + .copied() + .collect(); + modules.extend(self.client_main_modules().await?.iter().copied()); + Ok(Vc::cell(modules)) + } + + #[turbo_tasks::function] + pub async fn get_all_additional_entries( + self: Vc, + graphs: Vc, + ) -> Result> { + let mut modules: Vec>> = self + .get_all_endpoints() + .await? + .iter() + .map(|endpoint| endpoint.additional_root_modules(graphs)) + .try_flat_join() + .await? + .into_iter() + .copied() + .collect(); + modules.extend(self.client_main_modules().await?.iter().copied()); Ok(Vc::cell(modules)) } @@ -1469,16 +1490,22 @@ impl Project { /// Gets the module id strategy for the project. #[turbo_tasks::function] pub async fn module_id_strategy(self: Vc) -> Result>> { - let module_id_strategy = self.next_config().module_id_strategy_config(); - match *module_id_strategy.await? { - Some(ModuleIdStrategyConfig::Named) => Ok(Vc::upcast(DevModuleIdStrategy::new())), - Some(ModuleIdStrategyConfig::Deterministic) => { - Ok(Vc::upcast(GlobalModuleIdStrategyBuilder::build(self))) + let module_id_strategy = if let Some(module_id_strategy) = + &*self.next_config().module_id_strategy_config().await? + { + *module_id_strategy + } else { + match *self.next_mode().await? { + NextMode::Development => ModuleIdStrategyConfig::Named, + NextMode::Build => ModuleIdStrategyConfig::Deterministic, + } + }; + + match module_id_strategy { + ModuleIdStrategyConfig::Named => Ok(Vc::upcast(DevModuleIdStrategy::new())), + ModuleIdStrategyConfig::Deterministic => { + Ok(Vc::upcast(get_global_module_id_strategy(self))) } - None => match *self.next_mode().await? { - NextMode::Development => Ok(Vc::upcast(DevModuleIdStrategy::new())), - NextMode::Build => Ok(Vc::upcast(DevModuleIdStrategy::new())), - }, } } } diff --git a/crates/next-api/src/route.rs b/crates/next-api/src/route.rs index 35dbe27055e37..3281902cf340b 100644 --- a/crates/next-api/src/route.rs +++ b/crates/next-api/src/route.rs @@ -6,7 +6,7 @@ use turbo_tasks::{ }; use turbopack_core::module::Modules; -use crate::paths::ServerPath; +use crate::{module_graph::ReducedGraphs, paths::ServerPath}; #[derive(TraceRawVcs, Serialize, Deserialize, PartialEq, Eq, ValueDebugFormat, Clone, Debug)] pub struct AppPageRoute { @@ -63,9 +63,18 @@ pub trait Endpoint { fn write_to_disk(self: Vc) -> Vc; fn server_changed(self: Vc) -> Vc; fn client_changed(self: Vc) -> Vc; + /// The entry modules for the single modules graph. fn root_modules(self: Vc) -> Vc; + /// Additional entry modules for the single module graph. + /// This may read the single module graph and return additional modules. + fn additional_root_modules(self: Vc, _graphs: Vc) -> Vc { + Modules::empty() + } } +#[turbo_tasks::value(transparent, local)] +pub struct Endpoints(Vec>>); + #[turbo_tasks::value(shared)] #[derive(Debug, Clone)] pub enum WrittenEndpoint { diff --git a/crates/next-api/src/server_actions.rs b/crates/next-api/src/server_actions.rs index 26f4feb5925db..e991045155a42 100644 --- a/crates/next-api/src/server_actions.rs +++ b/crates/next-api/src/server_actions.rs @@ -29,6 +29,7 @@ use turbopack_core::{ chunk::{ChunkItem, ChunkItemExt, ChunkableModule, ChunkingContext, EvaluatableAsset}, context::AssetContext, file_source::FileSource, + ident::AssetIdent, module::Module, module_graph::SingleModuleGraph, output::OutputAsset, @@ -82,6 +83,11 @@ pub(crate) async fn create_server_actions_manifest( .cell()) } +#[turbo_tasks::function] +fn server_actions_loader_modifier() -> Vc { + Vc::cell("server actions loader".into()) +} + /// Builds the "action loader" entry point, which reexports every found action /// behind a lazy dynamic import. /// @@ -89,7 +95,7 @@ pub(crate) async fn create_server_actions_manifest( /// file's name and the action name). This hash matches the id sent to the /// client and present inside the paired manifest. #[turbo_tasks::function] -async fn build_server_actions_loader( +pub(crate) async fn build_server_actions_loader( project_path: Vc, page_name: RcStr, actions: Vc, @@ -114,10 +120,12 @@ async fn build_server_actions_loader( )?; } - let output_path = - project_path.join(format!(".next-internal/server/app{page_name}/actions.js").into()); + let path = project_path.join(format!(".next-internal/server/app{page_name}/actions.js").into()); let file = File::from(contents.build()); - let source = VirtualSource::new(output_path, AssetContent::file(file.into())); + let source = VirtualSource::new_with_ident( + AssetIdent::from_path(path).with_modifier(server_actions_loader_modifier()), + AssetContent::file(file.into()), + ); let import_map = import_map.into_iter().map(|(k, v)| (v, k)).collect(); let module = asset_context .process( diff --git a/crates/next-core/src/next_app/app_client_references_chunks.rs b/crates/next-core/src/next_app/app_client_references_chunks.rs index 6476dce1646c8..7bd3518a9b6ca 100644 --- a/crates/next-core/src/next_app/app_client_references_chunks.rs +++ b/crates/next-core/src/next_app/app_client_references_chunks.rs @@ -10,7 +10,6 @@ use turbopack_core::{ output::OutputAssets, }; -use super::include_modules_module::IncludeModulesModule; use crate::{ next_client_reference::{ visit_client_reference::ClientReferenceGraphResult, ClientReferenceType, @@ -24,8 +23,8 @@ pub fn client_modules_modifier() -> Vc { } #[turbo_tasks::function] -pub fn client_modules_ssr_modifier() -> Vc { - Vc::cell("client modules ssr".into()) +pub fn ssr_modules_modifier() -> Vc { + Vc::cell("ssr modules".into()) } #[turbo_tasks::value] @@ -202,13 +201,9 @@ pub async fn get_app_client_references_chunks( ) .entered(); - let ssr_entry_module = IncludeModulesModule::new( - base_ident.with_modifier(client_modules_ssr_modifier()), + ssr_chunking_context.chunk_group_multiple( + base_ident.with_modifier(ssr_modules_modifier()), ssr_modules, - ); - ssr_chunking_context.chunk_group( - ssr_entry_module.ident(), - Vc::upcast(ssr_entry_module), Value::new(current_ssr_availability_info), ) }) @@ -242,13 +237,9 @@ pub async fn get_app_client_references_chunks( ) .entered(); - let client_entry_module = IncludeModulesModule::new( + Some(client_chunking_context.chunk_group_multiple( base_ident.with_modifier(client_modules_modifier()), client_modules, - ); - Some(client_chunking_context.chunk_group( - client_entry_module.ident(), - Vc::upcast(client_entry_module), Value::new(current_client_availability_info), )) } else { diff --git a/crates/next-core/src/next_config.rs b/crates/next-core/src/next_config.rs index 54e8c2708322c..e3677dcf0eb55 100644 --- a/crates/next-core/src/next_config.rs +++ b/crates/next-core/src/next_config.rs @@ -566,7 +566,7 @@ pub enum LoaderItem { } #[turbo_tasks::value(operation)] -#[derive(Clone, Debug)] +#[derive(Copy, Clone, Debug)] #[serde(rename_all = "camelCase")] pub enum ModuleIdStrategy { Named, @@ -1469,11 +1469,11 @@ impl NextConfig { .experimental .turbo .as_ref() - .and_then(|t| t.module_id_strategy.as_ref()) + .and_then(|t| t.module_id_strategy) else { return Vc::cell(None); }; - Vc::cell(Some(module_id_strategy.clone())) + Vc::cell(Some(module_id_strategy)) } #[turbo_tasks::function] diff --git a/crates/next-core/src/next_server_component/server_component_module.rs b/crates/next-core/src/next_server_component/server_component_module.rs index 35b566d97743f..6de21531cfe8c 100644 --- a/crates/next-core/src/next_server_component/server_component_module.rs +++ b/crates/next-core/src/next_server_component/server_component_module.rs @@ -9,7 +9,7 @@ use turbopack_core::{ asset::{Asset, AssetContent}, chunk::{ChunkItem, ChunkItemExt, ChunkType, ChunkableModule, ChunkingContext}, ident::AssetIdent, - module::{Module, Modules}, + module::Module, reference::ModuleReferences, }; use turbopack_ecmascript::{ @@ -17,18 +17,11 @@ use turbopack_ecmascript::{ EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkPlaceable, EcmascriptChunkType, EcmascriptExports, }, - references::{ - esm::{EsmExport, EsmExports}, - external_module::IncludeIdentModule, - }, + references::esm::{EsmExport, EsmExports}, utils::StringifyJs, }; use super::server_component_reference::NextServerComponentModuleReference; -use crate::next_app::app_client_references_chunks::{ - client_modules_modifier, client_modules_ssr_modifier, -}; - #[turbo_tasks::function] fn modifier() -> Vc { Vc::cell("Next.js server component".into()) @@ -67,22 +60,6 @@ impl Module for NextServerComponentModule { .await?, )])) } - - #[turbo_tasks::function] - async fn additional_layers_modules(self: Vc) -> Result> { - let base_ident = self.ident(); - let ssr_entry_module = ResolvedVc::upcast( - IncludeIdentModule::new(base_ident.with_modifier(client_modules_ssr_modifier())) - .to_resolved() - .await?, - ); - let client_entry_module = ResolvedVc::upcast( - IncludeIdentModule::new(base_ident.with_modifier(client_modules_modifier())) - .to_resolved() - .await?, - ); - Ok(Vc::cell(vec![ssr_entry_module, client_entry_module])) - } } #[turbo_tasks::value_impl] diff --git a/test/e2e/app-dir/build-size/index.test.ts b/test/e2e/app-dir/build-size/index.test.ts index 527c17afd222c..029307568dd35 100644 --- a/test/e2e/app-dir/build-size/index.test.ts +++ b/test/e2e/app-dir/build-size/index.test.ts @@ -33,13 +33,12 @@ describe('app-dir build size', () => { const index = result['/'] const foo = result['/foo'] - // index route has a page, so it should not be 0 - expect(sizeToBytes(index.size)).toBeGreaterThan(0) expect(sizeToBytes(index.firstLoadJS)).toBeGreaterThan(0) + expect(sizeToBytes(foo.firstLoadJS)).toBeGreaterThan(0) - // foo route has a page, so it should not be 0 + // index route has a page with no client JS, so it could serve zero additional JS (size = 0) + // foo route has a page with client references, so has to serve additional non-shared JS expect(sizeToBytes(foo.size)).toBeGreaterThan(0) - expect(sizeToBytes(foo.firstLoadJS)).toBeGreaterThan(0) // foo is a client component, so it should be larger than index expect(sizeToBytes(foo.size)).toBeGreaterThan(sizeToBytes(index.size)) diff --git a/test/e2e/app-dir/non-root-project-monorepo/non-root-project-monorepo.test.ts b/test/e2e/app-dir/non-root-project-monorepo/non-root-project-monorepo.test.ts index 0a8bb94ba5f2b..74c42a2a455e0 100644 --- a/test/e2e/app-dir/non-root-project-monorepo/non-root-project-monorepo.test.ts +++ b/test/e2e/app-dir/non-root-project-monorepo/non-root-project-monorepo.test.ts @@ -112,7 +112,7 @@ describe('non-root-project-monorepo', () => { expect(normalizeStackTrace(await getRedboxCallStack(browser))) .toMatchInlineSnapshot(` " - [project]/apps/web/app/separate-file.ts [app-rsc] (ecmascript) (rsc://React/Server/file:///apps/web/.next/server/chunks/ssr/apps_web_8d1c0a._.js (7:7) + [project]/apps/web/app/separate-file.ts [app-rsc] (ecmascript) (rsc://React/Server/file:///apps/web/.next/server/chunks/ssr/apps_web_b7421c._.js (7:7) innerFunction app/source-maps-rsc/page.tsx (10:3) Page diff --git a/turbopack/crates/turbopack-browser/src/chunking_context.rs b/turbopack/crates/turbopack-browser/src/chunking_context.rs index 5ce366e04052e..9a10610ddfe11 100644 --- a/turbopack/crates/turbopack-browser/src/chunking_context.rs +++ b/turbopack/crates/turbopack-browser/src/chunking_context.rs @@ -389,10 +389,11 @@ impl ChunkingContext for BrowserChunkingContext { } #[turbo_tasks::function] - async fn chunk_group( + + async fn chunk_group_multiple( self: ResolvedVc, ident: Vc, - module: ResolvedVc>, + modules: Vec>>, availability_info: Value, ) -> Result> { let span = tracing::info_span!("chunking", ident = ident.to_string().await?.to_string()); @@ -404,7 +405,7 @@ impl ChunkingContext for BrowserChunkingContext { availability_info, } = make_chunk_group( ResolvedVc::upcast(self), - [ResolvedVc::upcast(module)], + modules.into_iter().map(ResolvedVc::upcast), input_availability_info, ) .await?; diff --git a/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs b/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs index b3f1695a19874..a954afaf5cd29 100644 --- a/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs +++ b/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs @@ -123,6 +123,15 @@ pub trait ChunkingContext { ident: Vc, module: Vc>, availability_info: Value, + ) -> Vc { + self.chunk_group_multiple(ident, vec![module], availability_info) + } + + fn chunk_group_multiple( + self: Vc, + ident: Vc, + modules: Vec>>, + availability_info: Value, ) -> Vc; fn evaluated_chunk_group( diff --git a/turbopack/crates/turbopack-core/src/chunk/module_id_strategies.rs b/turbopack/crates/turbopack-core/src/chunk/module_id_strategies.rs index 6bc1b537b1f27..7a6418aa03c4e 100644 --- a/turbopack/crates/turbopack-core/src/chunk/module_id_strategies.rs +++ b/turbopack/crates/turbopack-core/src/chunk/module_id_strategies.rs @@ -4,7 +4,10 @@ use turbo_tasks::{FxIndexMap, ResolvedVc, ValueToString, Vc}; use turbo_tasks_hash::hash_xxh3_hash64; use super::ModuleId; -use crate::ident::AssetIdent; +use crate::{ + ident::AssetIdent, + issue::{module::ModuleIssue, IssueExt, StyledString}, +}; #[turbo_tasks::value_trait] pub trait ModuleIdStrategy { @@ -47,10 +50,30 @@ impl GlobalModuleIdStrategy { impl ModuleIdStrategy for GlobalModuleIdStrategy { #[turbo_tasks::function] async fn get_module_id(&self, ident: Vc) -> Result> { - let ident_string = ident.to_string().await?.clone_value(); - if let Some(module_id) = self.module_id_map.get(&ident_string) { + let ident_string = ident.to_string().await?; + if let Some(module_id) = self.module_id_map.get(&*ident_string) { return Ok(module_id.clone().cell()); } + + if !ident_string.ends_with("[app-client] (ecmascript, next/dynamic entry)") { + // TODO: This shouldn't happen, but is a temporary workaround to ignore next/dynamic + // imports of a server component from another server component. + + ModuleIssue { + ident: ident.to_resolved().await?, + title: StyledString::Text( + format!("ModuleId not found for ident: {:?}", ident_string).into(), + ) + .resolved_cell(), + description: StyledString::Text( + format!("ModuleId not found for ident: {:?}", ident_string).into(), + ) + .resolved_cell(), + } + .resolved_cell() + .emit(); + } + Ok(ModuleId::String( hash_xxh3_hash64(ident.to_string().await?) .to_string() diff --git a/turbopack/crates/turbopack-core/src/module.rs b/turbopack/crates/turbopack-core/src/module.rs index 1b1429a7389ef..31b408c84710f 100644 --- a/turbopack/crates/turbopack-core/src/module.rs +++ b/turbopack/crates/turbopack-core/src/module.rs @@ -16,10 +16,6 @@ pub trait Module: Asset { ModuleReferences::empty() } - fn additional_layers_modules(self: Vc) -> Vc { - Vc::cell(vec![]) - } - /// Signifies the module itself is async, e.g. it uses top-level await, is a wasm module, etc. fn is_self_async(self: Vc) -> Vc { Vc::cell(false) diff --git a/turbopack/crates/turbopack-core/src/module_graph/mod.rs b/turbopack/crates/turbopack-core/src/module_graph/mod.rs index d4d5953cc9ecd..2c699b78116b4 100644 --- a/turbopack/crates/turbopack-core/src/module_graph/mod.rs +++ b/turbopack/crates/turbopack-core/src/module_graph/mod.rs @@ -7,7 +7,7 @@ use std::{ use anyhow::{Context, Result}; use petgraph::{ graph::{DiGraph, EdgeIndex, NodeIndex}, - visit::{Dfs, VisitMap, Visitable}, + visit::{Dfs, EdgeRef, VisitMap, Visitable}, }; use serde::{Deserialize, Serialize}; use turbo_rcstr::RcStr; @@ -85,17 +85,19 @@ impl SingleModuleGraph { { let _span = tracing::info_span!("build module graph").entered(); for (parent, current) in children_nodes_iter.into_breadth_first_edges() { - let parent_edge = parent.map(|parent| match parent { - SingleModuleGraphBuilderNode::Module { module, .. } => { - (*modules.get(&module).unwrap(), COMMON_CHUNKING_TYPE) + let parent = if let Some(parent) = parent { + match parent { + SingleModuleGraphBuilderNode::Module { module, .. } => { + Some(*modules.get(&module).unwrap()) + } + // was already handled in the previous iteration + SingleModuleGraphBuilderNode::ChunkableReference { .. } => continue, + // should never have children anyway + SingleModuleGraphBuilderNode::Issues { .. } => unreachable!(), } - SingleModuleGraphBuilderNode::ChunkableReference { - source, - chunking_type, - .. - } => (*modules.get(&source).unwrap(), chunking_type), - SingleModuleGraphBuilderNode::Issues { .. } => unreachable!(), - }); + } else { + None + }; match current { SingleModuleGraphBuilderNode::Module { @@ -116,16 +118,35 @@ impl SingleModuleGraph { idx }; // Add the edge - if let Some((parent_idx, chunking_type)) = parent_edge { - graph.add_edge(parent_idx, current_idx, chunking_type); + if let Some(parent_idx) = parent { + graph.add_edge(parent_idx, current_idx, COMMON_CHUNKING_TYPE); } } - SingleModuleGraphBuilderNode::ChunkableReference { .. } => { - // Ignore. They are handled when visiting the next edge - // (ChunkableReference -> Module) + SingleModuleGraphBuilderNode::ChunkableReference { + target, + target_layer, + chunking_type, + .. + } => { + // Handle them right now, because there might not be a child module if it + // was already visited in `visited_modules`. + // Find the target node, if it was already added + let target_idx = if let Some(target_idx) = modules.get(&target) { + *target_idx + } else { + let idx = graph.add_node(SingleModuleGraphNode { + module: target, + issues: Default::default(), + layer: target_layer, + }); + modules.insert(target, idx); + idx + }; + let parent_idx = parent.unwrap(); + graph.add_edge(parent_idx, target_idx, chunking_type); } SingleModuleGraphBuilderNode::Issues(new_issues) => { - let (parent_idx, _) = parent_edge.unwrap(); + let parent_idx = parent.unwrap(); graph .node_weight_mut(parent_idx) .unwrap() @@ -261,6 +282,45 @@ impl SingleModuleGraph { Ok(()) } + /// Traverses all edges exactly once and calls the visitor with the edge source and + /// target. + /// + /// This means that target nodes can be revisited (once per incoming edge). + pub fn traverse_edges<'a>( + &'a self, + mut visitor: impl FnMut( + ( + Option<(&'a SingleModuleGraphNode, &'a ChunkingType)>, + &'a SingleModuleGraphNode, + ), + ) -> GraphTraversalAction, + ) -> Result<()> { + let graph = &self.graph; + let mut stack = self.entries.values().copied().collect::>(); + let mut discovered = graph.visit_map(); + for entry_node in self.entries.values() { + let entry_weight = graph.node_weight(*entry_node).unwrap(); + visitor((None, entry_weight)); + } + + while let Some(node) = stack.pop() { + let node_weight = graph.node_weight(node).unwrap(); + if discovered.visit(node) { + for edge in graph.edges(node).collect::>() { + let edge_weight = edge.weight(); + let succ = edge.target(); + let succ_weight = graph.node_weight(succ).unwrap(); + let action = visitor((Some((node_weight, edge_weight)), succ_weight)); + if !discovered.is_visited(&succ) && action == GraphTraversalAction::Continue { + stack.push(succ); + } + } + } + } + + Ok(()) + } + /// Traverses all reachable edges in topological order. The preorder visitor can be used to /// forward state down the graph, and to skip subgraphs /// @@ -359,9 +419,17 @@ impl SingleModuleGraph { SingleModuleGraph::new_inner(None, &*entries.await?, &Default::default()).await } - /// `root` is connected to the entries and include in `self.entries`. #[turbo_tasks::function] pub async fn new_with_entries_visited( + entries: Vc, + visited_modules: Vc, + ) -> Result> { + SingleModuleGraph::new_inner(None, &*entries.await?, &*visited_modules.await?).await + } + + /// `root` is connected to the entries and include in `self.entries`. + #[turbo_tasks::function] + pub async fn new_with_entries_visited_root( root: ResolvedVc>, // This must not be a Vc> to ensure layout segment optimization hits the cache entries: Vec>>, @@ -442,6 +510,7 @@ enum SingleModuleGraphBuilderNode { source_ident: ReadRef, target: ResolvedVc>, target_ident: ReadRef, + target_layer: Option>, }, Module { module: ResolvedVc>, @@ -470,12 +539,17 @@ impl SingleModuleGraphBuilderNode { target: ResolvedVc>, chunking_type: ChunkingType, ) -> Result { + let target_ident = target.ident(); Ok(Self::ChunkableReference { chunking_type, source, source_ident: source.ident().to_string().await?, target, - target_ident: target.ident().to_string().await?, + target_ident: target_ident.to_string().await?, + target_layer: match target_ident.await?.layer { + Some(layer) => Some(layer.await?), + None => None, + }, }) } } diff --git a/turbopack/crates/turbopack-ecmascript/src/async_chunk/module.rs b/turbopack/crates/turbopack-ecmascript/src/async_chunk/module.rs index 8b8ef50e169b4..34f2a823a5dbb 100644 --- a/turbopack/crates/turbopack-ecmascript/src/async_chunk/module.rs +++ b/turbopack/crates/turbopack-ecmascript/src/async_chunk/module.rs @@ -12,7 +12,7 @@ use turbopack_core::{ use crate::async_chunk::chunk_item::AsyncLoaderChunkItem; #[turbo_tasks::function] -fn modifier() -> Vc { +pub fn async_loader_modifier() -> Vc { Vc::cell("async loader".into()) } @@ -42,7 +42,7 @@ impl AsyncLoaderModule { #[turbo_tasks::function] pub fn asset_ident_for(module: Vc>) -> Vc { - module.ident().with_modifier(modifier()) + module.ident().with_modifier(async_loader_modifier()) } } diff --git a/turbopack/crates/turbopack-ecmascript/src/global_module_id_strategy.rs b/turbopack/crates/turbopack-ecmascript/src/global_module_id_strategy.rs index 1b80379853b38..8021dcebef91e 100644 --- a/turbopack/crates/turbopack-ecmascript/src/global_module_id_strategy.rs +++ b/turbopack/crates/turbopack-ecmascript/src/global_module_id_strategy.rs @@ -1,185 +1,14 @@ use anyhow::Result; use turbo_rcstr::RcStr; -use turbo_tasks::{ - graph::{AdjacencyMap, GraphTraversal}, - FxIndexMap, FxIndexSet, ResolvedVc, TryJoinIterExt, ValueToString, Vc, -}; +use turbo_tasks::{FxIndexMap, FxIndexSet}; use turbo_tasks_hash::hash_xxh3_hash64; -use turbopack_core::{ - chunk::ModuleId, - module::{Module, Modules}, - reference::ModuleReference, -}; - -use crate::references::esm::EsmAsyncAssetReference; - -#[turbo_tasks::value] -pub struct PreprocessedChildrenIdents { - // ident.to_string() -> full hash - // We save the full hash to avoid re-hashing in `merge_preprocessed_module_ids` - // if this endpoint did not change. - modules_idents: FxIndexMap, -} - -#[derive(Clone, Hash)] -#[turbo_tasks::value(shared)] -pub enum ReferencedModule { - Module(ResolvedVc>), - AsyncLoaderModule(ResolvedVc>), -} - -impl ReferencedModule { - fn module(&self) -> Vc> { - match *self { - ReferencedModule::Module(module) => *module, - ReferencedModule::AsyncLoaderModule(module) => *module, - } - } -} - -#[turbo_tasks::value(transparent)] -pub struct ReferencedModules(Vec>); - -#[turbo_tasks::function] -async fn referenced_modules(module: Vc>) -> Result> { - let references = module.references().await?; - - // TODO(LichuAcu): Reduce type complexity - #[allow(clippy::type_complexity)] - type ModulesAndAsyncLoaders = Vec<( - Vec>>, - Option>>, - )>; - let modules_and_async_loaders: ModulesAndAsyncLoaders = references - .iter() - .map(|reference| async move { - let async_loader = - if ResolvedVc::try_downcast_type::(*reference) - .await? - .is_some() - { - *reference - .resolve_reference() - .resolve() - .await? - .first_module() - .await? - } else { - None - }; - - let modules = reference - .resolve_reference() - .resolve() - .await? - .primary_modules() - .await? - .clone_value(); - - Ok((modules, async_loader)) - }) - .try_join() - .await?; - - let mut set = FxIndexSet::default(); - let mut modules = Vec::new(); - for (module_list, async_loader) in modules_and_async_loaders { - for module in module_list { - if set.insert(module) { - modules.push(ReferencedModule::Module(module).resolved_cell()); - } - } - if let Some(async_loader_module) = async_loader { - if set.insert(async_loader_module) { - modules - .push(ReferencedModule::AsyncLoaderModule(async_loader_module).resolved_cell()); - } - } - } - - Ok(Vc::cell(modules)) -} - -pub async fn get_children_modules( - parent: ResolvedVc, -) -> Result> + Send> { - let parent_module = parent.await?.module(); - let mut modules = referenced_modules(parent_module).await?.clone_value(); - for module in parent_module.additional_layers_modules().await? { - modules.push(ReferencedModule::Module(*module).resolved_cell()); - } - Ok(modules.into_iter()) -} - -// NOTE(LichuAcu) Called on endpoint.root_modules(). It would probably be better if this was called -// directly on `Endpoint`, but such struct is not available in turbopack-core. The whole function -// could be moved to `next-api`, but it would require adding turbo-tasks-hash to `next-api`, -// making it heavier. -#[turbo_tasks::function] -pub async fn children_modules_idents( - root_modules: Vc, -) -> Result> { - let children_modules_iter = AdjacencyMap::new() - .skip_duplicates() - .visit( - root_modules - .await? - .iter() - .map(|module| ReferencedModule::Module(*module).resolved_cell()) - .collect::>(), - get_children_modules, - ) - .await - .completed()? - .into_inner() - .into_reverse_topological(); - - // module_id -> full hash - let mut modules_idents = FxIndexMap::default(); - for child_module in children_modules_iter { - match *child_module.await? { - ReferencedModule::Module(module) => { - let module_ident = module.ident(); - let ident_str = module_ident.to_string().await?.clone_value(); - let hash = hash_xxh3_hash64(&ident_str); - modules_idents.insert(ident_str, hash); - } - ReferencedModule::AsyncLoaderModule(async_loader_module) => { - let loader_ident = async_loader_module - .ident() - .with_modifier(Vc::cell("async loader".into())); - let loader_ident_str = loader_ident.to_string().await?.clone_value(); - let loader_hash = hash_xxh3_hash64(&loader_ident_str); - modules_idents.insert(loader_ident_str, loader_hash); - - let loaded_client_ident = async_loader_module - .ident() - .with_layer(Vc::cell("app-client".into())); - let loaded_client_ident_str = loaded_client_ident.to_string().await?.clone_value(); - let loaded_client_hash = hash_xxh3_hash64(&loaded_client_ident_str); - modules_idents.insert(loaded_client_ident_str, loaded_client_hash); - } - } - } - - Ok(PreprocessedChildrenIdents { modules_idents }.cell()) -} +use turbopack_core::chunk::ModuleId; const JS_MAX_SAFE_INTEGER: u64 = (1u64 << 53) - 1; -// Note(LichuAcu): This could be split into two functions: one that merges the preprocessed module -// ids and another that generates the final, optimized module ids. Thoughts? pub async fn merge_preprocessed_module_ids( - preprocessed_module_ids: Vec>, + merged_module_ids: &FxIndexMap, ) -> Result> { - let mut merged_module_ids = FxIndexMap::default(); - - for preprocessed_module_ids in preprocessed_module_ids { - for (module_ident, full_hash) in &preprocessed_module_ids.await?.modules_idents { - merged_module_ids.insert(module_ident.clone(), *full_hash); - } - } - // 5% fill rate, as done in Webpack // https://github.com/webpack/webpack/blob/27cf3e59f5f289dfc4d76b7a1df2edbc4e651589/lib/ids/IdHelpers.js#L366-L405 let optimal_range = merged_module_ids.len() * 20; diff --git a/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs b/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs index efa790ad6a26e..27c762820d64f 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs @@ -273,7 +273,7 @@ impl EcmascriptChunkItem for CachedExternalModuleChunkItem { /// /// It is used to include a module's ident in the module graph before the module /// itself is resolved, as is the case with NextServerComponentModule's -/// "client modules" and "client modules ssr". +/// "client modules" and "ssr modules". #[turbo_tasks::value] pub struct IncludeIdentModule { ident: ResolvedVc, diff --git a/turbopack/crates/turbopack-ecmascript/src/tree_shake/chunk_item.rs b/turbopack/crates/turbopack-ecmascript/src/tree_shake/chunk_item.rs index b9856e1da4087..162a882fe54d2 100644 --- a/turbopack/crates/turbopack-ecmascript/src/tree_shake/chunk_item.rs +++ b/turbopack/crates/turbopack-ecmascript/src/tree_shake/chunk_item.rs @@ -1,8 +1,10 @@ use anyhow::Result; -use turbo_tasks::{ResolvedVc, ValueDefault, ValueToString, Vc}; +use turbo_tasks::{ResolvedVc, ValueDefault, Vc}; use turbo_tasks_fs::rope::RopeBuilder; use turbopack_core::{ - chunk::{AsyncModuleInfo, ChunkItem, ChunkType, ChunkingContext}, + chunk::{ + AsyncModuleInfo, ChunkItem, ChunkItemExt, ChunkType, ChunkableModule, ChunkingContext, + }, ident::AssetIdent, module::Module, }; @@ -160,7 +162,12 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem { format!( "{}__turbopack_import__({});\n", if need_await { "await " } else { "" }, - StringifyJs(&*side_effect.ident().to_string().await?) + StringifyJs( + &*side_effect + .as_chunk_item(*self.chunking_context) + .id() + .await? + ) ) .as_bytes(), ); @@ -169,7 +176,13 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem { code.push_bytes( format!( "__turbopack_export_namespace__(__turbopack_import__({}));\n", - StringifyJs(&*module.resolved_as.ident().to_string().await?) + StringifyJs( + &*module + .resolved_as + .as_chunk_item(*self.chunking_context) + .id() + .await? + ) ) .as_bytes(), ); diff --git a/turbopack/crates/turbopack-nodejs/src/chunking_context.rs b/turbopack/crates/turbopack-nodejs/src/chunking_context.rs index 1e3e1329f7131..502a0f82b8990 100644 --- a/turbopack/crates/turbopack-nodejs/src/chunking_context.rs +++ b/turbopack/crates/turbopack-nodejs/src/chunking_context.rs @@ -292,23 +292,20 @@ impl ChunkingContext for NodeJsChunkingContext { } #[turbo_tasks::function] - async fn chunk_group( + async fn chunk_group_multiple( self: ResolvedVc, - _ident: Vc, - module: ResolvedVc>, + ident: Vc, + modules: Vec>>, availability_info: Value, ) -> Result> { - let span = tracing::info_span!( - "chunking", - module = module.ident().to_string().await?.to_string() - ); + let span = tracing::info_span!("chunking", module = ident.to_string().await?.to_string()); async move { let MakeChunkGroupResult { chunks, availability_info, } = make_chunk_group( ResolvedVc::upcast(self), - [ResolvedVc::upcast(module)], + modules.into_iter().map(ResolvedVc::upcast), availability_info.into_value(), ) .await?; From 53a3ccb73af607e061ed666c0e89f6049bde594a Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:44:46 +0100 Subject: [PATCH 02/11] Parallelize --- crates/next-api/src/module_graph.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/next-api/src/module_graph.rs b/crates/next-api/src/module_graph.rs index 1421e04e5ea8f..ea154c1b12413 100644 --- a/crates/next-api/src/module_graph.rs +++ b/crates/next-api/src/module_graph.rs @@ -504,8 +504,7 @@ impl ReducedGraphs { .try_join() .await } - .instrument(tracing::info_span!("generating next/dynamic graphs")) - .await?; + .instrument(tracing::info_span!("generating next/dynamic graphs")); let server_actions = async { graphs @@ -516,8 +515,7 @@ impl ReducedGraphs { .try_join() .await } - .instrument(tracing::info_span!("generating server actions graphs")) - .await?; + .instrument(tracing::info_span!("generating server actions graphs")); let client_references = async { graphs @@ -528,13 +526,15 @@ impl ReducedGraphs { .try_join() .await } - .instrument(tracing::info_span!("generating client references graphs")) - .await?; + .instrument(tracing::info_span!("generating client references graphs")); + + let (next_dynamic, server_actions, client_references) = + futures::join!(next_dynamic, server_actions, client_references); Ok(Self { - next_dynamic, - server_actions, - client_references, + next_dynamic: next_dynamic?, + server_actions: server_actions?, + client_references: client_references?, } .cell()) } From e9e3a1e4e399b048816edba73d2aca81f87405cc Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:56:21 +0100 Subject: [PATCH 03/11] Wrap modules argument in Vc --- crates/next-api/src/app.rs | 6 +++--- .../src/next_app/app_client_references_chunks.rs | 10 +++++----- .../crates/turbopack-browser/src/chunking_context.rs | 7 ++++--- .../turbopack-core/src/chunk/chunking_context.rs | 8 ++++---- turbopack/crates/turbopack-core/src/chunk/mod.rs | 3 +++ .../crates/turbopack-nodejs/src/chunking_context.rs | 7 ++++--- 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/crates/next-api/src/app.rs b/crates/next-api/src/app.rs index f603ebf484f30..123d1c2f29db6 100644 --- a/crates/next-api/src/app.rs +++ b/crates/next-api/src/app.rs @@ -1525,9 +1525,9 @@ impl AppEndpoint { .server_utils .iter() .map(|m| async move { - Ok(*ResolvedVc::try_downcast::>(*m) + ResolvedVc::try_downcast::>(*m) .await? - .context("Expected server utils to be chunkable")?) + .context("Expected server utils to be chunkable") }) .try_join() .await?; @@ -1537,7 +1537,7 @@ impl AppEndpoint { this.app_project.project().project_path(), ) .with_modifier(server_utils_modifier()), - server_utils, + Vc::cell(server_utils), Value::new(current_availability_info), ) .await?; diff --git a/crates/next-core/src/next_app/app_client_references_chunks.rs b/crates/next-core/src/next_app/app_client_references_chunks.rs index 7bd3518a9b6ca..6b03fed8e9886 100644 --- a/crates/next-core/src/next_app/app_client_references_chunks.rs +++ b/crates/next-core/src/next_app/app_client_references_chunks.rs @@ -183,7 +183,7 @@ pub async fn get_app_client_references_chunks( let ecmascript_client_reference_ref = ecmascript_client_reference.await?; - Some(*ResolvedVc::upcast( + Some(ResolvedVc::upcast( ecmascript_client_reference_ref.ssr_module, )) } @@ -203,7 +203,7 @@ pub async fn get_app_client_references_chunks( ssr_chunking_context.chunk_group_multiple( base_ident.with_modifier(ssr_modules_modifier()), - ssr_modules, + Vc::cell(ssr_modules), Value::new(current_ssr_availability_info), ) }) @@ -221,10 +221,10 @@ pub async fn get_app_client_references_chunks( } => { let ecmascript_client_reference_ref = ecmascript_client_reference.await?; - *ResolvedVc::upcast(ecmascript_client_reference_ref.client_module) + ResolvedVc::upcast(ecmascript_client_reference_ref.client_module) } ClientReferenceType::CssClientReference(css_module) => { - *ResolvedVc::upcast(*css_module) + ResolvedVc::upcast(*css_module) } }) }) @@ -239,7 +239,7 @@ pub async fn get_app_client_references_chunks( Some(client_chunking_context.chunk_group_multiple( base_ident.with_modifier(client_modules_modifier()), - client_modules, + Vc::cell(client_modules), Value::new(current_client_availability_info), )) } else { diff --git a/turbopack/crates/turbopack-browser/src/chunking_context.rs b/turbopack/crates/turbopack-browser/src/chunking_context.rs index 9a10610ddfe11..3160c4f2aa0b4 100644 --- a/turbopack/crates/turbopack-browser/src/chunking_context.rs +++ b/turbopack/crates/turbopack-browser/src/chunking_context.rs @@ -8,7 +8,7 @@ use turbopack_core::{ availability_info::AvailabilityInfo, chunk_group::{make_chunk_group, MakeChunkGroupResult}, module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy}, - Chunk, ChunkGroupResult, ChunkItem, ChunkableModule, ChunkingContext, + Chunk, ChunkGroupResult, ChunkItem, ChunkableModule, ChunkableModules, ChunkingContext, EntryChunkGroupResult, EvaluatableAssets, MinifyType, ModuleId, }, environment::Environment, @@ -393,19 +393,20 @@ impl ChunkingContext for BrowserChunkingContext { async fn chunk_group_multiple( self: ResolvedVc, ident: Vc, - modules: Vec>>, + modules: Vc, availability_info: Value, ) -> Result> { let span = tracing::info_span!("chunking", ident = ident.to_string().await?.to_string()); async move { let this = self.await?; + let modules = modules.await?; let input_availability_info = availability_info.into_value(); let MakeChunkGroupResult { chunks, availability_info, } = make_chunk_group( ResolvedVc::upcast(self), - modules.into_iter().map(ResolvedVc::upcast), + modules.iter().copied().map(ResolvedVc::upcast), input_availability_info, ) .await?; diff --git a/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs b/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs index a954afaf5cd29..6072064ce2e9d 100644 --- a/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs +++ b/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs @@ -7,7 +7,7 @@ use turbo_tasks_hash::DeterministicHash; use super::{availability_info::AvailabilityInfo, ChunkableModule, EvaluatableAssets}; use crate::{ - chunk::{ChunkItem, ModuleId}, + chunk::{ChunkItem, ChunkableModules, ModuleId}, environment::Environment, ident::AssetIdent, module::Module, @@ -121,16 +121,16 @@ pub trait ChunkingContext { fn chunk_group( self: Vc, ident: Vc, - module: Vc>, + module: ResolvedVc>, availability_info: Value, ) -> Vc { - self.chunk_group_multiple(ident, vec![module], availability_info) + self.chunk_group_multiple(ident, Vc::cell(vec![module]), availability_info) } fn chunk_group_multiple( self: Vc, ident: Vc, - modules: Vec>>, + modules: Vc, availability_info: Value, ) -> Vc; diff --git a/turbopack/crates/turbopack-core/src/chunk/mod.rs b/turbopack/crates/turbopack-core/src/chunk/mod.rs index 3523ea619ee0f..9c6a8f16ce89b 100644 --- a/turbopack/crates/turbopack-core/src/chunk/mod.rs +++ b/turbopack/crates/turbopack-core/src/chunk/mod.rs @@ -93,6 +93,9 @@ pub trait ChunkableModule: Module + Asset { ) -> Vc>; } +#[turbo_tasks::value(transparent)] +pub struct ChunkableModules(Vec>>); + #[turbo_tasks::value(transparent)] pub struct Chunks(Vec>>); diff --git a/turbopack/crates/turbopack-nodejs/src/chunking_context.rs b/turbopack/crates/turbopack-nodejs/src/chunking_context.rs index 502a0f82b8990..5edc3ff982596 100644 --- a/turbopack/crates/turbopack-nodejs/src/chunking_context.rs +++ b/turbopack/crates/turbopack-nodejs/src/chunking_context.rs @@ -10,7 +10,7 @@ use turbopack_core::{ availability_info::AvailabilityInfo, chunk_group::{make_chunk_group, MakeChunkGroupResult}, module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy}, - Chunk, ChunkGroupResult, ChunkItem, ChunkableModule, ChunkingContext, + Chunk, ChunkGroupResult, ChunkItem, ChunkableModule, ChunkableModules, ChunkingContext, EntryChunkGroupResult, EvaluatableAssets, MinifyType, ModuleId, }, environment::Environment, @@ -295,17 +295,18 @@ impl ChunkingContext for NodeJsChunkingContext { async fn chunk_group_multiple( self: ResolvedVc, ident: Vc, - modules: Vec>>, + modules: Vc, availability_info: Value, ) -> Result> { let span = tracing::info_span!("chunking", module = ident.to_string().await?.to_string()); async move { + let modules = modules.await?; let MakeChunkGroupResult { chunks, availability_info, } = make_chunk_group( ResolvedVc::upcast(self), - modules.into_iter().map(ResolvedVc::upcast), + modules.iter().copied().map(ResolvedVc::upcast), availability_info.into_value(), ) .await?; From 1bb9e4216d08eabd62169382d61e151889176b17 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Wed, 8 Jan 2025 12:01:05 +0100 Subject: [PATCH 04/11] Isolate take_collectibles into separate function --- crates/next-api/src/module_graph.rs | 31 ++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/crates/next-api/src/module_graph.rs b/crates/next-api/src/module_graph.rs index ea154c1b12413..7a01189e9447d 100644 --- a/crates/next-api/src/module_graph.rs +++ b/crates/next-api/src/module_graph.rs @@ -718,20 +718,37 @@ pub async fn get_reduced_graphs_for_endpoint( } #[turbo_tasks::function] -pub async fn get_global_module_id_strategy( +async fn get_module_graph_for_project_without_issues( project: Vc, -) -> Result> { +) -> Result> { let graph_op = get_module_graph_for_project(project); // TODO get rid of this once everything inside calls `take_collectibles()` when needed - let graph = graph_op.strongly_consistent().await?; + let graph = graph_op.resolve_strongly_consistent().await?; let _ = graph_op.take_collectibles::>(); + Ok(graph) +} - let additional_graph_op = get_additional_module_graph_for_project(project, graph_op); +#[turbo_tasks::function] +async fn get_additional_module_graph_for_project_without_issues( + project: Vc, + graph: Vc, +) -> Result> { + let graph_op = get_additional_module_graph_for_project(project, graph); // TODO get rid of this once everything inside calls `take_collectibles()` when needed - let additional_graph = additional_graph_op.strongly_consistent().await?; - let _ = additional_graph_op.take_collectibles::>(); + let graph = graph_op.resolve_strongly_consistent().await?; + let _ = graph_op.take_collectibles::>(); + Ok(graph) +} + +#[turbo_tasks::function] +pub async fn get_global_module_id_strategy( + project: Vc, +) -> Result> { + let graph = get_module_graph_for_project_without_issues(project); + let additional_graph = + get_additional_module_graph_for_project_without_issues(project, graph).await?; - let graphs = [graph, additional_graph]; + let graphs = [graph.await?, additional_graph]; let mut idents: Vec<_> = graphs .iter() From a5da6f2815720ec385a6085ad573ffd22fc90e96 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:04:54 +0100 Subject: [PATCH 05/11] Cleanup --- crates/next-api/src/module_graph.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/next-api/src/module_graph.rs b/crates/next-api/src/module_graph.rs index 7a01189e9447d..dbf04a7c825a2 100644 --- a/crates/next-api/src/module_graph.rs +++ b/crates/next-api/src/module_graph.rs @@ -1,6 +1,7 @@ use std::{ borrow::Cow, collections::{HashMap, HashSet}, + future::IntoFuture, }; use anyhow::Result; @@ -670,7 +671,8 @@ async fn get_reduced_graphs_for_endpoint_inner_operation( let (is_single_page, graphs) = match &*project.next_mode().await? { NextMode::Development => ( true, - async move { get_module_graph_for_endpoint(*entry).await } + get_module_graph_for_endpoint(*entry) + .into_future() .instrument(tracing::info_span!("module graph for endpoint")) .await? .iter() @@ -680,17 +682,15 @@ async fn get_reduced_graphs_for_endpoint_inner_operation( NextMode::Build => ( false, vec![ - *async move { - get_module_graph_for_project(*project) - // This is a performance optimization. This function is a root aggregation - // function that aggregates over the whole subgraph. - .resolve_strongly_consistent() - .await? - .to_resolved() - .await - } - .instrument(tracing::info_span!("module graph for app")) - .await?, + *get_module_graph_for_project(*project) + // This is a performance optimization. This function is a root aggregation + // function that aggregates over the whole subgraph. + .resolve_strongly_consistent() + .into_future() + .instrument(tracing::info_span!("module graph for app")) + .await? + .to_resolved() + .await?, ], ), }; From 61bbea95cede097fbbc66d5c7b43a7b11dae3cd6 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:38:59 +0100 Subject: [PATCH 06/11] Workaround backend bug --- .../crates/turbopack-browser/src/chunking_context.rs | 9 +++++++++ .../turbopack-core/src/chunk/chunking_context.rs | 6 ++---- .../crates/turbopack-nodejs/src/chunking_context.rs | 10 ++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/turbopack/crates/turbopack-browser/src/chunking_context.rs b/turbopack/crates/turbopack-browser/src/chunking_context.rs index 3160c4f2aa0b4..a55f6593a1a31 100644 --- a/turbopack/crates/turbopack-browser/src/chunking_context.rs +++ b/turbopack/crates/turbopack-browser/src/chunking_context.rs @@ -389,7 +389,16 @@ impl ChunkingContext for BrowserChunkingContext { } #[turbo_tasks::function] + fn chunk_group( + self: Vc, + ident: Vc, + module: ResolvedVc>, + availability_info: Value, + ) -> Vc { + self.chunk_group_multiple(ident, Vc::cell(vec![module]), availability_info) + } + #[turbo_tasks::function] async fn chunk_group_multiple( self: ResolvedVc, ident: Vc, diff --git a/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs b/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs index 6072064ce2e9d..b9b726edf4350 100644 --- a/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs +++ b/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs @@ -121,11 +121,9 @@ pub trait ChunkingContext { fn chunk_group( self: Vc, ident: Vc, - module: ResolvedVc>, + module: Vc>, availability_info: Value, - ) -> Vc { - self.chunk_group_multiple(ident, Vc::cell(vec![module]), availability_info) - } + ) -> Vc; fn chunk_group_multiple( self: Vc, diff --git a/turbopack/crates/turbopack-nodejs/src/chunking_context.rs b/turbopack/crates/turbopack-nodejs/src/chunking_context.rs index 5edc3ff982596..30c0a9d5d3bef 100644 --- a/turbopack/crates/turbopack-nodejs/src/chunking_context.rs +++ b/turbopack/crates/turbopack-nodejs/src/chunking_context.rs @@ -291,6 +291,16 @@ impl ChunkingContext for NodeJsChunkingContext { Ok(self.asset_root_path.join(asset_path.into())) } + #[turbo_tasks::function] + fn chunk_group( + self: Vc, + ident: Vc, + module: ResolvedVc>, + availability_info: Value, + ) -> Vc { + self.chunk_group_multiple(ident, Vc::cell(vec![module]), availability_info) + } + #[turbo_tasks::function] async fn chunk_group_multiple( self: ResolvedVc, From 87f25b4883944b45089f55a2637c1ee1a27881bd Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Fri, 10 Jan 2025 14:33:40 +0100 Subject: [PATCH 07/11] Relax test assertion --- .../non-root-project-monorepo.test.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/e2e/app-dir/non-root-project-monorepo/non-root-project-monorepo.test.ts b/test/e2e/app-dir/non-root-project-monorepo/non-root-project-monorepo.test.ts index 74c42a2a455e0..58ba766a22ae2 100644 --- a/test/e2e/app-dir/non-root-project-monorepo/non-root-project-monorepo.test.ts +++ b/test/e2e/app-dir/non-root-project-monorepo/non-root-project-monorepo.test.ts @@ -109,10 +109,14 @@ describe('non-root-project-monorepo', () => { 16 |" `) // TODO stacktrace-parser breaks in some cases with the rsc:// protocol - expect(normalizeStackTrace(await getRedboxCallStack(browser))) - .toMatchInlineSnapshot(` + expect( + normalizeStackTrace(await getRedboxCallStack(browser)).replace( + /\/apps_web_\w+._.js /, + '/apps_web_XXXXXX._.js ' + ) + ).toMatchInlineSnapshot(` " - [project]/apps/web/app/separate-file.ts [app-rsc] (ecmascript) (rsc://React/Server/file:///apps/web/.next/server/chunks/ssr/apps_web_b7421c._.js (7:7) + [project]/apps/web/app/separate-file.ts [app-rsc] (ecmascript) (rsc://React/Server/file:///apps/web/.next/server/chunks/ssr/apps_web_XXXXXX._.js (7:7) innerFunction app/source-maps-rsc/page.tsx (10:3) Page From 2be2532eb14516fe76884c274c2742ba4976cc56 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Fri, 10 Jan 2025 14:39:57 +0100 Subject: [PATCH 08/11] Relax test assertion --- .../pages-dir/production/test/index.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/production/pages-dir/production/test/index.test.ts b/test/production/pages-dir/production/test/index.test.ts index 7a258f88df56d..f8b4e8c015c38 100644 --- a/test/production/pages-dir/production/test/index.test.ts +++ b/test/production/pages-dir/production/test/index.test.ts @@ -580,16 +580,16 @@ describe('Production Usage', () => { const resources: Set = new Set() - const manifestKey = Object.keys(reactLoadableManifest).find((item) => { - return item - .replace(/\\/g, '/') - .endsWith( - process.env.TURBOPACK - ? 'components/dynamic-css/with-css.js [client] (ecmascript, next/dynamic entry)' - : 'dynamic/css.js -> ../../components/dynamic-css/with-css' - ) - }) + console.log(reactLoadableManifest) + expect(Object.keys(reactLoadableManifest).length).toBe(1) + const manifestKey = Object.keys(reactLoadableManifest)[0] expect(manifestKey).toBeString() + if (!process.env.TURBOPACK) { + // the key is a non-deterministic number for Turbopack prod + expect(manifestKey).toEndWith( + 'dynamic/css.js -> ../../components/dynamic-css/with-css' + ) + } // test dynamic chunk reactLoadableManifest[manifestKey].files.forEach((f) => { From 3407f10f6d8dd81b732efaab7be3ca5170d5cbf2 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:53:01 +0100 Subject: [PATCH 09/11] Relax test assertion --- .../pages-dir/production/test/index.test.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/test/production/pages-dir/production/test/index.test.ts b/test/production/pages-dir/production/test/index.test.ts index f8b4e8c015c38..0e2b58d707f15 100644 --- a/test/production/pages-dir/production/test/index.test.ts +++ b/test/production/pages-dir/production/test/index.test.ts @@ -580,14 +580,17 @@ describe('Production Usage', () => { const resources: Set = new Set() - console.log(reactLoadableManifest) - expect(Object.keys(reactLoadableManifest).length).toBe(1) - const manifestKey = Object.keys(reactLoadableManifest)[0] - expect(manifestKey).toBeString() - if (!process.env.TURBOPACK) { - // the key is a non-deterministic number for Turbopack prod - expect(manifestKey).toEndWith( - 'dynamic/css.js -> ../../components/dynamic-css/with-css' + let manifestKey: string + if (process.env.TURBOPACK) { + // the key is a non-deterministic number for Turbopack prod, but each page has its own manifest + expect(Object.keys(reactLoadableManifest).length).toBe(1) + manifestKey = Object.keys(reactLoadableManifest)[0] + expect(manifestKey).toBeString() + } else { + manifestKey = Object.keys(reactLoadableManifest).find((item) => + item + .replace(/\\/g, '/') + .endsWith('dynamic/css.js -> ../../components/dynamic-css/with-css') ) } From d2c8fbbed9f14e7153eb8ce5f68353b4ace3f1f3 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:01:26 +0100 Subject: [PATCH 10/11] Fix URL encoding of client reference chunks --- .../src/next_manifests/client_reference_manifest.rs | 4 ++-- turbopack/crates/turbo-tasks-fs/src/util.rs | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/next-core/src/next_manifests/client_reference_manifest.rs b/crates/next-core/src/next_manifests/client_reference_manifest.rs index ecb7f1c884e32..1f89aa92cce96 100644 --- a/crates/next-core/src/next_manifests/client_reference_manifest.rs +++ b/crates/next-core/src/next_manifests/client_reference_manifest.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result}; use indoc::formatdoc; use turbo_rcstr::RcStr; use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, Value, ValueToString, Vc}; -use turbo_tasks_fs::{File, FileSystemPath}; +use turbo_tasks_fs::{util::uri_from_file_relative, File, FileSystemPath}; use turbopack_core::{ asset::{Asset, AssetContent}, chunk::{ @@ -100,10 +100,10 @@ impl ClientReferenceManifest { let chunk_paths = client_chunks_paths .iter() .filter_map(|chunk_path| client_relative_path.get_path_to(chunk_path)) - .map(ToString::to_string) // It's possible that a chunk also emits CSS files, that will // be handled separatedly. .filter(|path| path.ends_with(".js")) + .map(uri_from_file_relative) .map(RcStr::from) .collect::>(); diff --git a/turbopack/crates/turbo-tasks-fs/src/util.rs b/turbopack/crates/turbo-tasks-fs/src/util.rs index d079b21468157..942182ae09933 100644 --- a/turbopack/crates/turbo-tasks-fs/src/util.rs +++ b/turbopack/crates/turbo-tasks-fs/src/util.rs @@ -147,7 +147,7 @@ pub async fn uri_from_file(root: Vc, path: Option<&str>) -> Resu Ok(format!( "file://{}", - &sys_to_unix( + &uri_from_file_relative(&sys_to_unix( &root_fs .to_sys_path(match path { Some(path) => root.join(path.into()), @@ -155,10 +155,13 @@ pub async fn uri_from_file(root: Vc, path: Option<&str>) -> Resu }) .await? .to_string_lossy() - ) - .split('/') - .map(|s| urlencoding::encode(s)) + )) + )) +} + +pub fn uri_from_file_relative(path: &str) -> String { + path.split("/") + .map(|p| urlencoding::encode(p)) .collect::>() .join("/") - )) } From a0f19025cbfd9b912aa67d15784a28a15738f48d Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:24:41 +0100 Subject: [PATCH 11/11] No Turbopack pings --- packages/next/src/client/dev/on-demand-entries-client.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/next/src/client/dev/on-demand-entries-client.ts b/packages/next/src/client/dev/on-demand-entries-client.ts index cea83ee899229..32099c5c16d43 100644 --- a/packages/next/src/client/dev/on-demand-entries-client.ts +++ b/packages/next/src/client/dev/on-demand-entries-client.ts @@ -2,6 +2,12 @@ import Router from '../router' import { sendMessage } from '../components/react-dev-overlay/pages/websocket' export default async (page?: string) => { + // Never send pings when using Turbopack as it's not used. + // Pings were originally used to keep track of active routes in on-demand-entries with webpack. + if (process.env.TURBOPACK) { + return + } + if (page) { // in AMP the router isn't initialized on the client and // client-transitions don't occur so ping initial page