set up options / hooks to enable turbotrace (#72306)

Adds tracing settings to the various contexts that are used for nft

There are cases where we need to know whether tracing is useful, this PR sets up that state ahead of the next one(s) which actually take advantage of them.

Closes PACK-3377

crates/next-api/src/app.rs

@@ -73,6 +73,7 @@ use crate::{
     },
     font::create_font_manifest,
     loadable_manifest::create_react_loadable_manifest,
+    nft_json::NftJsonAsset,
     paths::{
         all_paths_in_root, all_server_paths, get_js_paths_from_root, get_paths_from_root,
         get_wasm_paths_from_root, paths_to_bindings, wasm_paths_to_bindings,
@@ -1140,29 +1141,34 @@ impl AppEndpoint {
         let app_entry_chunks_ref = app_entry_chunks.await?;
         server_assets.extend(app_entry_chunks_ref.iter().copied());
 
-        if let (Some(client_references), Some(client_references_chunks)) =
-            (client_references, client_references_chunks)
-        {
-            let entry_manifest = ClientReferenceManifest::build_output(
-                node_root,
-                client_relative_path,
-                app_entry.original_name.clone(),
-                client_references,
-                client_references_chunks,
-                *app_entry_chunks,
-                Value::new(*app_entry_chunks_availability),
-                client_chunking_context,
-                ssr_chunking_context,
-                this.app_project.project().next_config(),
-                runtime,
-            )
-            .to_resolved()
-            .await?;
-            server_assets.insert(entry_manifest);
-            if runtime == NextRuntime::Edge {
-                middleware_assets.push(entry_manifest);
-            }
-        }
+        // these references are important for turbotrace
+        let client_reference_manifest =
+            if let (Some(client_references), Some(client_references_chunks)) =
+                (client_references, client_references_chunks)
+            {
+                let entry_manifest = ClientReferenceManifest::build_output(
+                    node_root,
+                    client_relative_path,
+                    app_entry.original_name.clone(),
+                    client_references,
+                    client_references_chunks,
+                    *app_entry_chunks,
+                    Value::new(*app_entry_chunks_availability),
+                    client_chunking_context,
+                    ssr_chunking_context,
+                    this.app_project.project().next_config(),
+                    runtime,
+                )
+                .to_resolved()
+                .await?;
+                server_assets.insert(entry_manifest);
+                if runtime == NextRuntime::Edge {
+                    middleware_assets.push(entry_manifest);
+                }
+                Some(entry_manifest)
+            } else {
+                None
+            };
 
         let client_assets = OutputAssets::new(client_assets.iter().map(|asset| **asset).collect());
 
@@ -1348,6 +1354,26 @@ impl AppEndpoint {
                 );
                 server_assets.extend(loadable_manifest_output.await?.iter().copied());
 
+                if this
+                    .app_project
+                    .project()
+                    .next_mode()
+                    .await?
+                    .is_production()
+                {
+                    server_assets.insert(ResolvedVc::upcast(
+                        NftJsonAsset::new(
+                            *rsc_chunk,
+                            this.app_project.project().output_fs(),
+                            this.app_project.project().project_fs(),
+                            this.app_project.project().client_fs(),
+                            client_reference_manifest.iter().map(|m| **m).collect(),
+                        )
+                        .to_resolved()
+                        .await?,
+                    ));
+                }
+
                 AppEndpointOutput::NodeJs {
                     rsc_chunk,
                     server_assets: Vc::cell(server_assets.iter().cloned().collect::<Vec<_>>()),

crates/next-core/src/next_server/context.rs

@@ -524,6 +524,11 @@ pub async fn get_server_module_options_context(
         },
         tree_shaking_mode: tree_shaking_mode_for_user_code,
         side_effect_free_packages: next_config.optimize_package_imports().await?.clone_value(),
+        enable_externals_tracing: if next_mode.is_production() {
+            Some(project_path)
+        } else {
+            None
+        },
         ..Default::default()
     };
 
@@ -965,7 +970,8 @@ pub async fn get_server_chunking_context_with_client_assets(
     } else {
         MinifyType::NoMinify
     })
-    .module_id_strategy(module_id_strategy);
+    .module_id_strategy(module_id_strategy)
+    .file_tracing(next_mode.is_production());
 
     if next_mode.is_development() {
         builder = builder.use_file_source_map_uris();
@@ -1000,7 +1006,8 @@ pub async fn get_server_chunking_context(
     } else {
         MinifyType::NoMinify
     })
-    .module_id_strategy(module_id_strategy);
+    .module_id_strategy(module_id_strategy)
+    .file_tracing(next_mode.is_production());
 
     if next_mode.is_development() {
         builder = builder.use_file_source_map_uris()

turbopack/crates/turbopack-browser/src/chunking_context.rs

@@ -48,6 +48,11 @@ impl BrowserChunkingContextBuilder {
         self
     }
 
+    pub fn tracing(mut self, enable_tracing: bool) -> Self {
+        self.chunking_context.enable_tracing = enable_tracing;
+        self
+    }
+
     pub fn asset_base_path(mut self, asset_base_path: Vc<Option<RcStr>>) -> Self {
         self.chunking_context.asset_base_path = asset_base_path;
         self
@@ -128,6 +133,8 @@ pub struct BrowserChunkingContext {
     asset_base_path: Vc<Option<RcStr>>,
     /// Enable HMR for this chunking
     enable_hot_module_replacement: bool,
+    /// Enable tracing for this chunking
+    enable_tracing: bool,
     /// The environment chunks will be evaluated in.
     environment: Vc<Environment>,
     /// The kind of runtime to include in the output.
@@ -164,6 +171,7 @@ impl BrowserChunkingContext {
                 chunk_base_path: Default::default(),
                 asset_base_path: Default::default(),
                 enable_hot_module_replacement: false,
+                enable_tracing: false,
                 environment,
                 runtime_type,
                 minify_type: MinifyType::NoMinify,
@@ -363,6 +371,11 @@ impl ChunkingContext for BrowserChunkingContext {
         Vc::cell(self.should_use_file_source_map_uris)
     }
 
+    #[turbo_tasks::function]
+    fn is_tracing_enabled(&self) -> Vc<bool> {
+        Vc::cell(self.enable_tracing)
+    }
+
     #[turbo_tasks::function]
     async fn chunk_group(
         self: Vc<Self>,

turbopack/crates/turbopack-core/src/chunk/chunking_context.rs

@@ -84,6 +84,10 @@ pub trait ChunkingContext {
         Vc::cell(false)
     }
 
+    fn is_tracing_enabled(self: Vc<Self>) -> Vc<bool> {
+        Vc::cell(false)
+    }
+
     fn async_loader_chunk_item(
         &self,
         module: Vc<Box<dyn ChunkableModule>>,

turbopack/crates/turbopack-nodejs/src/chunking_context.rs

@@ -44,6 +44,11 @@ impl NodeJsChunkingContextBuilder {
         self
     }
 
+    pub fn file_tracing(mut self, enable_tracing: bool) -> Self {
+        self.chunking_context.enable_file_tracing = enable_tracing;
+        self
+    }
+
     pub fn runtime_type(mut self, runtime_type: RuntimeType) -> Self {
         self.chunking_context.runtime_type = runtime_type;
         self
@@ -91,6 +96,8 @@ pub struct NodeJsChunkingContext {
     environment: Vc<Environment>,
     /// The kind of runtime to include in the output.
     runtime_type: RuntimeType,
+    /// Enable tracing for this chunking
+    enable_file_tracing: bool,
     /// Whether to minify resulting chunks
     minify_type: MinifyType,
     /// Whether to use manifest chunks for lazy compilation
@@ -120,6 +127,7 @@ impl NodeJsChunkingContext {
                 chunk_root_path,
                 asset_root_path,
                 asset_prefix: Default::default(),
+                enable_file_tracing: false,
                 environment,
                 runtime_type,
                 minify_type: MinifyType::NoMinify,
@@ -201,6 +209,11 @@ impl ChunkingContext for NodeJsChunkingContext {
         self.environment
     }
 
+    #[turbo_tasks::function]
+    fn is_tracing_enabled(&self) -> Vc<bool> {
+        Vc::cell(self.enable_file_tracing)
+    }
+
     #[turbo_tasks::function]
     async fn asset_url(self: Vc<Self>, ident: Vc<AssetIdent>) -> Result<Vc<RcStr>> {
         let this = self.await?;

turbopack/crates/turbopack/src/lib.rs

@@ -319,6 +319,9 @@ pub struct ModuleAssetContext {
     pub resolve_options_context: Vc<ResolveOptionsContext>,
     pub layer: Vc<RcStr>,
     transition: Option<ResolvedVc<Box<dyn Transition>>>,
+    /// Whether to replace external resolutions with CachedExternalModules. Used with
+    /// ModuleOptionsContext.enable_externals_tracing to handle transitive external dependencies.
+    replace_externals: bool,
 }
 
 #[turbo_tasks::value_impl]
@@ -338,6 +341,7 @@ impl ModuleAssetContext {
             resolve_options_context,
             transition: None,
             layer,
+            replace_externals: true,
         })
     }
 
@@ -357,6 +361,26 @@ impl ModuleAssetContext {
             resolve_options_context,
             layer,
             transition: Some(transition),
+            replace_externals: true,
+        })
+    }
+
+    #[turbo_tasks::function]
+    fn new_without_replace_externals(
+        transitions: Vc<TransitionOptions>,
+        compile_time_info: Vc<CompileTimeInfo>,
+        module_options_context: Vc<ModuleOptionsContext>,
+        resolve_options_context: Vc<ResolveOptionsContext>,
+        layer: Vc<RcStr>,
+    ) -> Vc<Self> {
+        Self::cell(ModuleAssetContext {
+            transitions,
+            compile_time_info,
+            module_options_context,
+            resolve_options_context,
+            transition: None,
+            layer,
+            replace_externals: false,
         })
     }
 

turbopack/crates/turbopack/src/module_options/module_options_context.rs

@@ -1,5 +1,6 @@
 use serde::{Deserialize, Serialize};
 use turbo_tasks::{trace::TraceRawVcs, FxIndexMap, RcStr, ResolvedVc, ValueDefault, Vc};
+use turbo_tasks_fs::FileSystemPath;
 use turbopack_core::{
     chunk::MinifyType, condition::ContextCondition, environment::Environment,
     resolve::options::ImportMapping,
@@ -124,6 +125,13 @@ pub struct ModuleOptionsContext {
     pub side_effect_free_packages: Vec<RcStr>,
     pub tree_shaking_mode: Option<TreeShakingMode>,
 
+    /// Generate (non-emitted) output assets for static assets and externals, to facilitate
+    /// generating a list of all non-bundled files that will be required at runtime.
+    ///
+    /// The filepath is the directory from which the bundled files will require the externals at
+    /// runtime.
+    pub enable_externals_tracing: Option<Vc<FileSystemPath>>,
+
     /// Custom rules to be applied after all default rules.
     pub module_rules: Vec<ModuleRule>,
     /// A list of rules to use a different module option context for certain