Migrate global module ids to single-graph

crates/next-api/Cargo.toml

@@ -28,6 +28,7 @@ turbo-rcstr = { workspace = true }
 turbo-tasks = { workspace = true }
 turbo-tasks-env = { workspace = true }
 turbo-tasks-fs = { workspace = true }
+turbo-tasks-hash = { workspace = true }
 turbo-tasks-memory = { workspace = true }
 turbopack = { workspace = true }
 turbopack-browser = { workspace = true }

crates/next-api/src/lib.rs

@@ -9,7 +9,6 @@ mod dynamic_imports;
 mod empty;
 pub mod entrypoints;
 mod font;
-pub mod global_module_id_strategy;
 mod instrumentation;
 mod loadable_manifest;
 mod middleware;

crates/next-api/src/module_graph.rs

@@ -29,13 +29,15 @@ use turbo_tasks::{
     CollectiblesSource, FxIndexMap, FxIndexSet, ReadRef, ResolvedVc, TryFlatJoinIterExt,
     TryJoinIterExt, ValueToString, Vc,
 };
+use turbo_tasks_hash::hash_xxh3_hash64;
 use turbopack_core::{
-    chunk::ChunkingType,
+    chunk::{module_id_strategies::GlobalModuleIdStrategy, ChunkingType},
     context::AssetContext,
     issue::{Issue, IssueExt},
     module::{Module, Modules},
     reference::primary_chunkable_referenced_modules,
 };
+use turbopack_ecmascript::global_module_id_strategy::merge_preprocessed_module_ids;
 
 use crate::{
     client_references::{map_client_references, ClientReferenceMapType, ClientReferencesSet},
@@ -544,6 +546,11 @@ async fn get_module_graph_for_endpoint(
     Ok(Vc::cell(graphs))
 }
 
+#[turbo_tasks::function]
+async fn get_module_graph_for_project(project: ResolvedVc<Project>) -> Vc<SingleModuleGraph> {
+    SingleModuleGraph::new_with_entries(project.get_all_entries())
+}
+
 #[turbo_tasks::value]
 pub struct NextDynamicGraph {
     is_single_page: bool,
@@ -990,13 +997,9 @@ async fn get_reduced_graphs_for_endpoint_inner(
         NextMode::Build => (
             false,
             vec![
-                async move {
-                    SingleModuleGraph::new_with_entries(project.get_all_entries())
-                        .to_resolved()
-                        .await
-                }
-                .instrument(tracing::info_span!("module graph for app"))
-                .await?,
+                async move { get_module_graph_for_project(project).to_resolved().await }
+                    .instrument(tracing::info_span!("module graph for app"))
+                    .await?,
             ],
         ),
     };
@@ -1065,3 +1068,35 @@ pub async fn get_reduced_graphs_for_endpoint(
     }
     Ok(result)
 }
+
+/// Like get_reduced_graphs_for_endpoint, but may only be called for builds
+///
+/// If you can, use get_reduced_graphs_for_endpoint instead.
+#[turbo_tasks::function]
+pub async fn get_global_module_id_strategy(
+    project: Vc<Project>,
+) -> Result<Vc<GlobalModuleIdStrategy>> {
+    let graph_op = get_module_graph_for_project(project);
+    // TODO get rid of this function once everything inside of
+    // `get_reduced_graphs_for_endpoint_inner` calls `take_collectibles()` when needed
+    let graph = graph_op.strongly_consistent().await?;
+    let _ = graph_op.take_collectibles::<Box<dyn Issue>>();
+
+    let module_id_map: FxIndexMap<RcStr, u64> = graph
+        .iter_nodes()
+        .map(|node| async move {
+            let module_ident = node.module.ident();
+            let ident_str = module_ident.to_string().await?.clone_value();
+            let hash = hash_xxh3_hash64(&ident_str);
+            Ok((ident_str, hash))
+        })
+        .try_join()
+        .await?
+        .into_iter()
+        .collect();
+
+    // TODO clean up this call signature
+    let module_id_map = merge_preprocessed_module_ids(vec![Vc::cell(module_id_map)]).await?;
+
+    GlobalModuleIdStrategy::new(module_id_map).await
+}

crates/next-api/src/project.rs

@@ -64,9 +64,9 @@ use crate::{
     build,
     empty::EmptyEndpoint,
     entrypoints::Entrypoints,
-    global_module_id_strategy::GlobalModuleIdStrategyBuilder,
     instrumentation::InstrumentationEndpoint,
     middleware::MiddlewareEndpoint,
+    module_graph::get_global_module_id_strategy,
     pages::PagesProject,
     route::{AppPageRoute, Endpoint, Route},
     versioned_content_map::{OutputAssetsOperation, VersionedContentMap},
@@ -1393,16 +1393,22 @@ impl Project {
     /// Gets the module id strategy for the project.
     #[turbo_tasks::function]
     pub async fn module_id_strategy(self: Vc<Self>) -> Result<Vc<Box<dyn ModuleIdStrategy>>> {
-        let module_id_strategy = self.next_config().module_id_strategy_config();
-        match *module_id_strategy.await? {
-            Some(ModuleIdStrategyConfig::Named) => Ok(Vc::upcast(DevModuleIdStrategy::new())),
-            Some(ModuleIdStrategyConfig::Deterministic) => {
-                Ok(Vc::upcast(GlobalModuleIdStrategyBuilder::build(self)))
+        let module_id_strategy = if let Some(module_id_strategy) =
+            &*self.next_config().module_id_strategy_config().await?
+        {
+            *module_id_strategy
+        } else {
+            match *self.next_mode().await? {
+                NextMode::Development => ModuleIdStrategyConfig::Named,
+                NextMode::Build => ModuleIdStrategyConfig::Deterministic,
+            }
+        };
+
+        match module_id_strategy {
+            ModuleIdStrategyConfig::Named => Ok(Vc::upcast(DevModuleIdStrategy::new())),
+            ModuleIdStrategyConfig::Deterministic => {
+                Ok(Vc::upcast(get_global_module_id_strategy(self)))
             }
-            None => match *self.next_mode().await? {
-                NextMode::Development => Ok(Vc::upcast(DevModuleIdStrategy::new())),
-                NextMode::Build => Ok(Vc::upcast(DevModuleIdStrategy::new())),
-            },
         }
     }
 }

crates/next-core/src/next_config.rs

@@ -447,7 +447,7 @@ pub enum LoaderItem {
 }
 
 #[turbo_tasks::value]
-#[derive(Clone, Debug)]
+#[derive(Copy, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub enum ModuleIdStrategy {
     Named,
@@ -1296,11 +1296,11 @@ impl NextConfig {
             .experimental
             .turbo
             .as_ref()
-            .and_then(|t| t.module_id_strategy.as_ref())
+            .and_then(|t| t.module_id_strategy)
         else {
             return Vc::cell(None);
         };
-        Vc::cell(Some(module_id_strategy.clone()))
+        Vc::cell(Some(module_id_strategy))
     }
 
     #[turbo_tasks::function]

turbopack/crates/turbopack-core/src/chunk/module_id_strategies.rs

@@ -1,7 +1,6 @@
-use anyhow::Result;
+use anyhow::{bail, Result};
 use turbo_rcstr::RcStr;
 use turbo_tasks::{FxIndexMap, ResolvedVc, ValueToString, Vc};
-use turbo_tasks_hash::hash_xxh3_hash64;
 
 use super::ModuleId;
 use crate::ident::AssetIdent;
@@ -47,15 +46,19 @@ impl GlobalModuleIdStrategy {
 impl ModuleIdStrategy for GlobalModuleIdStrategy {
     #[turbo_tasks::function]
     async fn get_module_id(&self, ident: Vc<AssetIdent>) -> Result<Vc<ModuleId>> {
-        let ident_string = ident.to_string().await?.clone_value();
-        if let Some(module_id) = self.module_id_map.get(&ident_string) {
+        let ident_string = ident.to_string().await?;
+        if let Some(module_id) = self.module_id_map.get(&*ident_string) {
             return Ok(module_id.clone().cell());
         }
-        Ok(ModuleId::String(
-            hash_xxh3_hash64(ident.to_string().await?)
-                .to_string()
-                .into(),
-        )
-        .cell())
+        // TODO this shouldn't happen
+        // It means we missed something when generating the map
+        bail!("ModuleId not found for ident: {:?}", ident_string)
+
+        // Ok(ModuleId::String(
+        //     hash_xxh3_hash64(ident.to_string().await?)
+        //         .to_string()
+        //         .into(),
+        // )
+        // .cell())
     }
 }

turbopack/crates/turbopack-ecmascript/src/global_module_id_strategy.rs

@@ -13,13 +13,11 @@ use turbopack_core::{
 
 use crate::references::esm::EsmAsyncAssetReference;
 
-#[turbo_tasks::value]
-pub struct PreprocessedChildrenIdents {
-    // ident.to_string() -> full hash
-    // We save the full hash to avoid re-hashing in `merge_preprocessed_module_ids`
-    // if this endpoint did not change.
-    modules_idents: FxIndexMap<RcStr, u64>,
-}
+// ident.to_string() -> full hash
+// We save the full hash to avoid re-hashing in `merge_preprocessed_module_ids`
+// if this endpoint did not change.
+#[turbo_tasks::value(transparent)]
+pub struct PreprocessedChildrenIdents(FxIndexMap<RcStr, u64>);
 
 #[derive(Clone, Hash)]
 #[turbo_tasks::value(shared)]
@@ -104,10 +102,7 @@ pub async fn get_children_modules(
     parent: ResolvedVc<ReferencedModule>,
 ) -> Result<impl Iterator<Item = ResolvedVc<ReferencedModule>> + Send> {
     let parent_module = parent.await?.module();
-    let mut modules = referenced_modules(parent_module).await?.clone_value();
-    for module in parent_module.additional_layers_modules().await? {
-        modules.push(ReferencedModule::Module(*module).resolved_cell());
-    }
+    let modules = referenced_modules(parent_module).await?.clone_value();
     Ok(modules.into_iter())
 }
 
@@ -162,7 +157,7 @@ pub async fn children_modules_idents(
         }
     }
 
-    Ok(PreprocessedChildrenIdents { modules_idents }.cell())
+    Ok(Vc::cell(modules_idents))
 }
 
 const JS_MAX_SAFE_INTEGER: u64 = (1u64 << 53) - 1;
@@ -175,7 +170,7 @@ pub async fn merge_preprocessed_module_ids(
     let mut merged_module_ids = FxIndexMap::default();
 
     for preprocessed_module_ids in preprocessed_module_ids {
-        for (module_ident, full_hash) in &preprocessed_module_ids.await?.modules_idents {
+        for (module_ident, full_hash) in &preprocessed_module_ids.await? {
             merged_module_ids.insert(module_ident.clone(), *full_hash);
         }
     }