Remove esm loader (#55509)

Now that we are bundling the react runtime for app router we can remove
the esm loader as that was the reason it was added when bringing the
server back into a single process.

x-ref: #55362

---------

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

packages/next/src/bin/next.ts

@@ -5,14 +5,7 @@ import arg from 'next/dist/compiled/arg/index.js'
 import { NON_STANDARD_NODE_ENV } from '../lib/constants'
 import { commands } from '../lib/commands'
 import { commandArgs } from '../lib/command-args'
-import loadConfig from '../server/config'
-import {
-  PHASE_PRODUCTION_SERVER,
-  PHASE_DEVELOPMENT_SERVER,
-} from '../shared/lib/constants'
-import { getProjectDir } from '../lib/get-project-dir'
 import { getValidatedArgs } from '../lib/get-validated-args'
-import { findPagesDir } from '../lib/find-pages-dir'
 
 const defaultCommand = 'dev'
 const args = arg(
@@ -126,48 +119,6 @@ async function main() {
   const currentArgsSpec = commandArgs[command]()
   const validatedArgs = getValidatedArgs(currentArgsSpec, forwardedArgs)
 
-  if (
-    (command === 'start' || command === 'dev') &&
-    !process.env.NEXT_PRIVATE_WORKER
-  ) {
-    const dir = getProjectDir(
-      process.env.NEXT_PRIVATE_DEV_DIR || validatedArgs._[0]
-    )
-    process.env.NEXT_PRIVATE_DIR = dir
-    const origEnv = Object.assign({}, process.env)
-
-    // TODO: set config to env variable to be re-used so we don't reload
-    // un-necessarily
-    const config = await loadConfig(
-      command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_SERVER,
-      dir,
-      {
-        silent: false,
-      }
-    )
-    let dirsResult: ReturnType<typeof findPagesDir> | undefined = undefined
-
-    try {
-      dirsResult = findPagesDir(dir)
-    } catch (_) {
-      // handle this error further down
-    }
-
-    if (dirsResult?.appDir || process.env.NODE_ENV === 'development') {
-      process.env = origEnv
-    }
-
-    if (dirsResult?.appDir) {
-      // we need to reset env if we are going to create
-      // the worker process with the esm loader so that the
-      // initial env state is correct
-      process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = config.experimental
-        .serverActions
-        ? 'experimental'
-        : 'next'
-    }
-  }
-
   for (const dependency of ['react', 'react-dom']) {
     try {
       // When 'npm link' is used it checks the clone location. Not the project.

packages/next/src/build/index.ts

@@ -147,10 +147,9 @@ import { startTypeChecking } from './type-check'
 import { generateInterceptionRoutesRewrites } from '../lib/generate-interception-routes-rewrites'
 
 import { buildDataRoute } from '../server/lib/router-utils/build-data-route'
-import { defaultOverrides } from '../server/import-overrides'
+import { defaultOverrides } from '../server/require-hook'
 import { initialize as initializeIncrementalCache } from '../server/lib/incremental-cache-server'
 import { nodeFs } from '../server/lib/node-fs-methods'
-import { getEsmLoaderPath } from '../server/lib/get-esm-loader-path'
 
 export type SsgRoute = {
   initialRevalidateSeconds: number | false
@@ -1251,11 +1250,6 @@ export default async function build(
           },
           numWorkers,
           forkOptions: {
-            execArgv: [
-              '--experimental-loader',
-              getEsmLoaderPath(),
-              '--no-warnings',
-            ],
             env: {
               ...process.env,
               __NEXT_INCREMENTAL_CACHE_IPC_PORT: incrementalCacheIpcPort + '',
@@ -2135,8 +2129,7 @@ export default async function build(
                 ? [
                     require.resolve('next/dist/server/lib/start-server'),
                     require.resolve('next/dist/server/next'),
-                    require.resolve('next/dist/esm/server/esm-loader.mjs'),
-                    require.resolve('next/dist/server/import-overrides'),
+                    require.resolve('next/dist/server/require-hook'),
                   ]
                 : []),
               require.resolve('next/dist/server/next-server'),

packages/next/src/build/webpack-config.ts

@@ -69,7 +69,7 @@ import { NextFontManifestPlugin } from './webpack/plugins/next-font-manifest-plu
 import { getSupportedBrowsers } from './utils'
 import { MemoryWithGcCachePlugin } from './webpack/plugins/memory-with-gc-cache-plugin'
 import { getBabelConfigFile } from './get-babel-config-file'
-import { defaultOverrides } from '../server/import-overrides'
+import { defaultOverrides } from '../server/require-hook'
 
 type ExcludesFalse = <T>(x: T | false) => x is T
 type ClientEntries = {

packages/next/src/cli/next-dev.ts

@@ -17,23 +17,28 @@ import { fileExists, FileType } from '../lib/file-exists'
 import { getNpxCommand } from '../lib/helpers/get-npx-command'
 import { createSelfSignedCertificate } from '../lib/mkcert'
 import uploadTrace from '../trace/upload-trace'
-import { startServer } from '../server/lib/start-server'
-import { loadEnvConfig } from '@next/env'
+import { initialEnv, loadEnvConfig } from '@next/env'
 import { trace } from '../trace'
+import { validateTurboNextConfig } from '../lib/turbopack-warning'
+import { fork } from 'child_process'
+import { RESTART_EXIT_CODE } from '../server/lib/setup-server-worker'
 import {
   getReservedPortExplanation,
   isPortIsReserved,
 } from '../lib/helpers/get-reserved-port'
-import { validateTurboNextConfig } from '../lib/turbopack-warning'
 
 let dir: string
+let child: undefined | ReturnType<typeof fork>
 let config: NextConfigComplete
 let isTurboSession = false
 let traceUploadUrl: string
 let sessionStopHandled = false
 let sessionStarted = Date.now()
 
-const handleSessionStop = async () => {
+const handleSessionStop = async (signal: string | null) => {
+  if (child) {
+    child.kill((signal as any) || 0)
+  }
   if (sessionStopHandled) return
   sessionStopHandled = true
 
@@ -96,8 +101,8 @@ const handleSessionStop = async () => {
   process.exit(0)
 }
 
-process.on('SIGINT', handleSessionStop)
-process.on('SIGTERM', handleSessionStop)
+process.on('SIGINT', () => handleSessionStop('SIGINT'))
+process.on('SIGTERM', () => handleSessionStop('SIGTERM'))
 
 const nextDev: CliCommand = async (args) => {
   if (args['--help']) {
@@ -193,6 +198,14 @@ const nextDev: CliCommand = async (args) => {
     },
   })
 
+  process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = config.experimental
+    .serverActions
+    ? 'experimental'
+    : 'next'
+
+  // we need to reset env if we are going to create
+  // the worker process with the esm loader so that the
+  // initial env state is correct
   let envInfo: string[] = []
   if (loadedEnvFiles.length > 0) {
     envInfo = loadedEnvFiles.map((f) => f.path)
@@ -231,6 +244,42 @@ const nextDev: CliCommand = async (args) => {
   setGlobal('phase', PHASE_DEVELOPMENT_SERVER)
   setGlobal('distDir', distDir)
 
+  const startServerPath = require.resolve('../server/lib/start-server')
+  async function startServer(options: StartServerOptions) {
+    return new Promise<void>((resolve) => {
+      let resolved = false
+
+      child = fork(startServerPath, {
+        stdio: 'inherit',
+        env: {
+          ...((initialEnv || process.env) as typeof process.env),
+          NEXT_PRIVATE_WORKER: '1',
+        },
+      })
+
+      child.on('message', (msg: any) => {
+        if (msg && typeof msg === 'object') {
+          if (msg.nextWorkerReady) {
+            child?.send({ nextWorkerOptions: options })
+          } else if (msg.nextServerReady && !resolved) {
+            resolved = true
+            resolve()
+          }
+        }
+      })
+
+      child.on('exit', async (code, signal) => {
+        if (sessionStopHandled || signal) {
+          return
+        }
+        if (code === RESTART_EXIT_CODE) {
+          return startServer(options)
+        }
+        await handleSessionStop(signal)
+      })
+    })
+  }
+
   const runDevServer = async (reboot: boolean) => {
     try {
       if (!!args['--experimental-https']) {

packages/next/src/server/config-utils.ts

@@ -10,7 +10,7 @@ export function loadWebpackHook() {
 
   // hook the Node.js require so that webpack requires are
   // routed to the bundled and now initialized webpack version
-  require('../server/import-overrides').addHookAliases(
+  require('../server/require-hook').addHookAliases(
     [
       ['webpack', 'next/dist/compiled/webpack/webpack-lib'],
       ['webpack/package', 'next/dist/compiled/webpack/package'],

packages/next/src/server/config.ts

@@ -865,7 +865,7 @@ export default async function loadConfig(
 
       if (shouldExit) {
         for (const message of messages) {
-          curLog.error(message)
+          console.error(message)
         }
         await flushAndExit(1)
       } else {

packages/next/src/server/lib/router-server.ts

@@ -84,7 +84,8 @@ export async function initialize(opts: {
 
   const config = await loadConfig(
     opts.dev ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_SERVER,
-    opts.dir
+    opts.dir,
+    { silent: false }
   )
 
   let compress: ReturnType<typeof setupCompression> | undefined

packages/next/src/server/lib/start-server.ts

@@ -358,3 +358,13 @@ export async function startServer({
     })
   }
 }
+
+if (process.env.NEXT_PRIVATE_WORKER && process.send) {
+  process.addListener('message', async (msg: any) => {
+    if (msg && typeof msg && msg.nextWorkerOptions && process.send) {
+      await startServer(msg.nextWorkerOptions)
+      process.send({ nextServerReady: true })
+    }
+  })
+  process.send({ nextWorkerReady: true })
+}