Turbopack: Implement HMR in next-api (#54772)

by @jridgewell: ### What? This integrates Turbopack's HRM protocol within the existing HMR WebSocket, and implements the server-side of the Next's and Turbopack's protocols for use in next-api. ### Why? HMR makes the development experience. ### How? The (new) Turbopack HMR protocol allows reusing the existing `sendMessage` and `addMessageListener` API's already used by our HMR singleton in Pages. (App apparently doesn't have a per-chunk HMR, only per-page, so it's HMR signals are describe below.) For the next-api server-side, I implemented the following events: - `reloadPage`, for Pages when `_document` changes - `middlewareChanges` when middleware is added/deleted, or modified - `serverOnlyChanges` for Pages when a loaded page's server graph changes - `serverComponentChanges` for App when a loaded app page's server graph changes We reuse the already implemented `addedPage`, `removedPage`, and `devPagesManifestUpdate` (done via webpack, so we should eventually port that over to the Turbopack listeners). I don't know exactly where `built`, `building`, and `sync` should be integrated, so they're just not sent currently. Additionally, the client-sent events aren't implemented in the new HMR server. Depends on vercel/turborepo#5814 Closes WEB-1453 --------- Co-authored-by: Justin Ridgewell <[email protected]>
vercel · Aug 30, 2023 · 56c9ad8 · 56c9ad8
1 parent c8e0d57
commit 56c9ad8
Show file tree

Hide file tree

Showing 15 changed files with 363 additions and 157 deletions.
diff --git a/packages/next-swc/crates/next-api/src/middleware.rs b/packages/next-swc/crates/next-api/src/middleware.rs
@@ -214,6 +214,6 @@ impl Endpoint for MiddlewareEndpoint {
 
     #[turbo_tasks::function]
     fn client_changed(self: Vc<Self>) -> Vc<Completion> {
-        Completion::new()
+        Completion::immutable()
     }
 }
diff --git a/packages/next-swc/crates/next-api/src/project.rs b/packages/next-swc/crates/next-api/src/project.rs
@@ -586,7 +586,7 @@ impl Project {
         Ok(self
             .await?
             .versioned_content_map
-            .get(self.client_root().join(identifier)))
+            .get(self.client_relative_path().join(identifier)))
     }
 
     #[turbo_tasks::function]
@@ -635,7 +635,7 @@ impl Project {
         Ok(self
             .await?
             .versioned_content_map
-            .keys_in_path(self.client_root()))
+            .keys_in_path(self.client_relative_path()))
     }
 }
 

diff --git a/packages/next-swc/crates/next-core/src/next_client/context.rs b/packages/next-swc/crates/next-core/src/next_client/context.rs
@@ -136,7 +136,7 @@ pub async fn get_client_resolve_options_context(
     let next_client_import_map =
         get_next_client_import_map(project_path, ty, next_config, execution_context);
     let next_client_fallback_import_map = get_next_client_fallback_import_map(ty);
-    let next_client_resolved_map = get_next_client_resolved_map(project_path, project_path);
+    let next_client_resolved_map = get_next_client_resolved_map(project_path, project_path, mode);
     let module_options_context = ResolveOptionsContext {
         enable_node_modules: Some(project_path.root().resolve().await?),
         custom_conditions: vec![mode.node_env().to_string()],

diff --git a/packages/next-swc/crates/next-core/src/next_import_map.rs b/packages/next-swc/crates/next-core/src/next_import_map.rs
@@ -309,22 +309,28 @@ pub async fn get_next_edge_import_map(
 pub fn get_next_client_resolved_map(
     context: Vc<FileSystemPath>,
     root: Vc<FileSystemPath>,
+    mode: NextMode,
 ) -> Vc<ResolvedMap> {
-    let glob_mappings = vec![
-        // Temporary hack to replace the hot reloader until this is passable by props in next.js
-        (
-            context.root(),
-            Glob::new(
-                "**/next/dist/client/components/react-dev-overlay/hot-reloader-client.js"
-                    .to_string(),
+    let glob_mappings = if mode == NextMode::Development {
+        vec![]
+    } else {
+        vec![
+            // Temporary hack to replace the hot reloader until this is passable by props in
+            // next.js
+            (
+                context.root(),
+                Glob::new(
+                    "**/next/dist/client/components/react-dev-overlay/hot-reloader-client.js"
+                        .to_string(),
+                ),
+                ImportMapping::PrimaryAlternative(
+                    "@vercel/turbopack-next/dev/hot-reloader.tsx".to_string(),
+                    Some(root),
+                )
+                .into(),
             ),
-            ImportMapping::PrimaryAlternative(
-                "@vercel/turbopack-next/dev/hot-reloader.tsx".to_string(),
-                Some(root),
-            )
-            .into(),
-        ),
-    ];
+        ]
+    };
     ResolvedMap {
         by_glob: glob_mappings,
     }

diff --git a/packages/next/package.json b/packages/next/package.json
@@ -192,6 +192,7 @@
     "@types/ws": "8.2.0",
     "@vercel/ncc": "0.34.0",
     "@vercel/nft": "0.22.6",
+    "@vercel/turbopack-ecmascript-runtime": "https://gitpkg-fork.vercel.sh/vercel/turbo/crates/turbopack-ecmascript-runtime/js?turbopack-230829.2",
     "acorn": "8.5.0",
     "ajv": "8.11.0",
     "amphtml-validator": "1.0.35",

diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts
@@ -475,7 +475,7 @@ export type TurbopackResult<T = {}> = T & {
   diagnostics: Diagnostics[]
 }
 
-interface Middleware {
+export interface Middleware {
   endpoint: Endpoint
 }
 
@@ -500,7 +500,7 @@ export interface UpdateInfo {
   tasks: number
 }
 
-enum ServerClientChangeType {
+export enum ServerClientChangeType {
   Server = 'Server',
   Client = 'Client',
   Both = 'Both',
@@ -550,7 +550,7 @@ export interface Endpoint {
    * After changed() has been awaited it will listen to changes.
    * The async iterator will yield for each change.
    */
-  changed(): Promise<AsyncIterableIterator<TurbopackResult>>
+  changed(): Promise<AsyncIterableIterator<TurbopackResult<ServerClientChange>>>
 }
 
 interface EndpointConfig {
@@ -593,6 +593,8 @@ function bindingToApi(binding: any, _wasm: boolean) {
     callback: (err: Error, value: T) => void
   ) => Promise<{ __napiType: 'RootTask' }>
 
+  const cancel = new (class Cancel extends Error {})()
+
   /**
    * Utility function to ensure all variants of an enum are handled.
    */
@@ -635,6 +637,7 @@ function bindingToApi(binding: any, _wasm: boolean) {
           reject: (error: Error) => void
         }
       | undefined
+    let canceled = false
 
     // The native function will call this every time it emits a new result. We
     // either need to notify a waiting consumer, or buffer the new result until
@@ -652,10 +655,10 @@ function bindingToApi(binding: any, _wasm: boolean) {
       }
     }
 
-    return (async function* () {
+    const iterator = (async function* () {
       const task = await withErrorCause(() => nativeFunction(emitResult))
       try {
-        while (true) {
+        while (!canceled) {
           if (buffer.length > 0) {
             const item = buffer.shift()!
             if (item.err) throw item.err
@@ -667,10 +670,19 @@ function bindingToApi(binding: any, _wasm: boolean) {
             })
           }
         }
+      } catch (e) {
+        if (e === cancel) return
+        throw e
       } finally {
         binding.rootTaskDispose(task)
       }
     })()
+    iterator.return = async () => {
+      canceled = true
+      if (waiting) waiting.reject(cancel)
+      return { value: undefined, done: true } as IteratorReturnResult<never>
+    }
+    return iterator
   }
 
   /**
@@ -908,6 +920,10 @@ function bindingToApi(binding: any, _wasm: boolean) {
           )
       )
 
+      // The subscriptions will emit always emit once, which is the initial
+      // computation. This is not a change, so swallow it.
+      await Promise.all([serverSubscription.next(), clientSubscription.next()])
+
       return (async function* () {
         try {
           while (true) {

diff --git a/packages/next/src/client/dev/amp-dev.ts b/packages/next/src/client/dev/amp-dev.ts
@@ -77,14 +77,8 @@ async function tryApplyUpdates() {
   }
 }
 
-addMessageListener((event) => {
-  if (event.data === '\uD83D\uDC93') {
-    return
-  }
-
+addMessageListener((message) => {
   try {
-    const message = JSON.parse(event.data)
-
     // actions which are not related to amp-dev
     if (
       message.action === 'serverError' ||
@@ -102,7 +96,7 @@ addMessageListener((event) => {
     }
   } catch (err: any) {
     console.warn(
-      '[HMR] Invalid message: ' + event.data + '\n' + (err?.stack ?? '')
+      '[HMR] Invalid message: ' + message + '\n' + (err?.stack ?? '')
     )
   }
 })

diff --git a/packages/next/src/client/dev/dev-build-watcher.ts b/packages/next/src/client/dev/dev-build-watcher.ts
@@ -5,7 +5,7 @@ type VerticalPosition = 'top' | 'bottom'
 type HorizonalPosition = 'left' | 'right'
 
 export default function initializeBuildWatcher(
-  toggleCallback: (cb: (event: string | { data: string }) => void) => void,
+  toggleCallback: (cb: (obj: Record<string, any>) => void) => void,
   position = 'bottom-right'
 ) {
   const shadowHost = document.createElement('div')
@@ -53,21 +53,13 @@ export default function initializeBuildWatcher(
 
   // Handle events
 
-  addMessageListener((event) => {
-    // This is the heartbeat event
-    if (event.data === '\uD83D\uDC93') {
-      return
-    }
-
+  addMessageListener((obj) => {
     try {
-      handleMessage(event)
+      handleMessage(obj)
     } catch {}
   })
 
-  function handleMessage(event: string | { data: string }) {
-    const obj =
-      typeof event === 'string' ? { action: event } : JSON.parse(event.data)
-
+  function handleMessage(obj: Record<string, any>) {
     // eslint-disable-next-line default-case
     switch (obj.action) {
       case 'building':

diff --git a/packages/next/src/client/dev/error-overlay/hot-dev-client.ts b/packages/next/src/client/dev/error-overlay/hot-dev-client.ts
@@ -63,15 +63,14 @@ let customHmrEventHandler: any
 export default function connect() {
   register()
 
-  addMessageListener((event) => {
+  addMessageListener((payload) => {
     try {
-      const payload = JSON.parse(event.data)
-      if (!('action' in payload)) return
-
-      processMessage(payload)
+      if ('action' in payload) {
+        processMessage(payload)
+      }
     } catch (err: any) {
       console.warn(
-        '[HMR] Invalid message: ' + event.data + '\n' + (err?.stack ?? '')
+        '[HMR] Invalid message: ' + payload + '\n' + (err?.stack ?? '')
       )
     }
   })

diff --git a/packages/next/src/client/dev/error-overlay/websocket.ts b/packages/next/src/client/dev/error-overlay/websocket.ts
@@ -1,5 +1,7 @@
+type WebSocketMessage = Record<string, any>
+
 let source: WebSocket
-const eventCallbacks: ((event: any) => void)[] = []
+const eventCallbacks: ((msg: WebSocketMessage) => void)[] = []
 let lastActivity = Date.now()
 
 function getSocketProtocol(assetPrefix: string): string {
@@ -13,7 +15,7 @@ function getSocketProtocol(assetPrefix: string): string {
   return protocol === 'http:' ? 'ws' : 'wss'
 }
 
-export function addMessageListener(cb: (event: any) => void) {
+export function addMessageListener(cb: (msg: WebSocketMessage) => void) {
   eventCallbacks.push(cb)
 }
 
@@ -39,11 +41,17 @@ export function connectHMR(options: {
       lastActivity = Date.now()
     }
 
-    function handleMessage(event: any) {
+    function handleMessage(event: MessageEvent<string>) {
       lastActivity = Date.now()
 
+      // webpack's heartbeat event.
+      if (event.data === '\uD83D\uDC93') {
+        return
+      }
+
+      const msg = JSON.parse(event.data)
       eventCallbacks.forEach((cb) => {
-        cb(event)
+        cb(msg)
       })
     }
 

diff --git a/packages/next/src/client/next-dev-turbopack.ts b/packages/next/src/client/next-dev-turbopack.ts
@@ -1,8 +1,12 @@
 // TODO: Remove use of `any` type.
-import { initialize, hydrate, version, router, emitter } from './'
-import { displayContent } from './dev/fouc'
+import { initialize, version, router, emitter } from './'
+import initWebpackHMR from './dev/webpack-hot-middleware-client'
 
 import './setup-hydration-warning'
+import { pageBootrap } from './page-bootstrap'
+import { addMessageListener, sendMessage } from './dev/error-overlay/websocket'
+//@ts-expect-error requires "moduleResolution": "node16" in tsconfig.json and not .ts extension
+import { connect } from '@vercel/turbopack-ecmascript-runtime/dev/client/hmr-client.ts'
 
 window.next = {
   version: `${version}-turbo`,
@@ -17,11 +21,10 @@ window.next = {
 // for the page loader
 declare let __turbopack_load__: any
 
+const webpackHMR = initWebpackHMR()
 initialize({
   // TODO the prop name is confusing as related to webpack
-  webpackHMR: {
-    onUnrecoverableError() {},
-  },
+  webpackHMR,
 })
   .then(({ assetPrefix }) => {
     // for the page loader
@@ -51,7 +54,19 @@ initialize({
       )
     }
 
-    return hydrate({ beforeRender: displayContent }).then(() => {})
+    connect({
+      addMessageListener(cb: (msg: Record<string, string>) => void) {
+        addMessageListener((msg) => {
+          // Only call Turbopack's message listener for turbopack messages
+          if (msg.type?.startsWith('turbopack-')) {
+            cb(msg)
+          }
+        })
+      },
+      sendMessage,
+    })
+
+    return pageBootrap(assetPrefix)
   })
   .catch((err) => {
     console.error('Error was not caught', err)