Add hot-reloading for env file changes (#38483)

* Add hot-reloading for env file changes * update watching * update test * update initial env handling * undo test change * add comment for entries clearing * update on-demand-entry-handler restart handling * lint-fix * handle bad plugin
vercel · Aug 11, 2022 · 4e6d055 · 4e6d055
1 parent 970469f
commit 4e6d055
Show file tree

Hide file tree

Showing 13 changed files with 199 additions and 46 deletions.
diff --git a/packages/next-env/index.ts b/packages/next-env/index.ts
@@ -4,12 +4,13 @@ import * as path from 'path'
 import * as dotenv from 'dotenv'
 import { expand as dotenvExpand } from 'dotenv-expand'
 
-export type Env = { [key: string]: string }
+export type Env = { [key: string]: string | undefined }
 export type LoadedEnvFiles = Array<{
   path: string
   contents: string
 }>
 
+let initialEnv: Env | undefined = undefined
 let combinedEnv: Env | undefined = undefined
 let cachedLoadedEnvFiles: LoadedEnvFiles = []
 
@@ -21,18 +22,24 @@ type Log = {
 export function processEnv(
   loadedEnvFiles: LoadedEnvFiles,
   dir?: string,
-  log: Log = console
+  log: Log = console,
+  forceReload = false
 ) {
-  // don't reload env if we already have since this breaks escaped
-  // environment values e.g. \$ENV_FILE_KEY
-  if (process.env.__NEXT_PROCESSED_ENV || loadedEnvFiles.length === 0) {
+  if (!initialEnv) {
+    initialEnv = Object.assign({}, process.env)
+  }
+  // only reload env when forceReload is specified
+  if (
+    !forceReload &&
+    (process.env.__NEXT_PROCESSED_ENV || loadedEnvFiles.length === 0)
+  ) {
     return process.env as Env
   }
   // flag that we processed the environment values in case a serverless
   // function is re-used or we are running in `next start` mode
   process.env.__NEXT_PROCESSED_ENV = 'true'
 
-  const origEnv = Object.assign({}, process.env)
+  const origEnv = Object.assign({}, initialEnv)
   const parsed: dotenv.DotenvParseOutput = {}
 
   for (const envFile of loadedEnvFiles) {
@@ -61,21 +68,27 @@ export function processEnv(
       )
     }
   }
-
   return Object.assign(process.env, parsed)
 }
 
 export function loadEnvConfig(
   dir: string,
   dev?: boolean,
-  log: Log = console
+  log: Log = console,
+  forceReload = false
 ): {
   combinedEnv: Env
   loadedEnvFiles: LoadedEnvFiles
 } {
-  // don't reload env if we already have since this breaks escaped
-  // environment values e.g. \$ENV_FILE_KEY
-  if (combinedEnv) return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles }
+  if (!initialEnv) {
+    initialEnv = Object.assign({}, process.env)
+  }
+  // only reload env when forceReload is specified
+  if (combinedEnv && !forceReload) {
+    return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles }
+  }
+  process.env = Object.assign({}, initialEnv)
+  cachedLoadedEnvFiles = []
 
   const isTest = process.env.NODE_ENV === 'test'
   const mode = isTest ? 'test' : dev ? 'development' : 'production'
@@ -112,6 +125,6 @@ export function loadEnvConfig(
       }
     }
   }
-  combinedEnv = processEnv(cachedLoadedEnvFiles, dir, log)
+  combinedEnv = processEnv(cachedLoadedEnvFiles, dir, log, forceReload)
   return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles }
 }
diff --git a/packages/next/server/base-server.ts b/packages/next/server/base-server.ts
@@ -265,7 +265,10 @@ export default abstract class Server<ServerOptions extends Options = Options> {
     res: BaseNextResponse
   ): void
 
-  protected abstract loadEnvConfig(params: { dev: boolean }): void
+  protected abstract loadEnvConfig(params: {
+    dev: boolean
+    forceReload?: boolean
+  }): void
 
   public constructor(options: ServerOptions) {
     const {

diff --git a/packages/next/server/dev/hot-reloader.ts b/packages/next/server/dev/hot-reloader.ts
@@ -31,6 +31,7 @@ import { findPageFile } from '../lib/find-page-file'
 import {
   BUILDING,
   entries,
+  getInvalidator,
   onDemandEntryHandler,
 } from './on-demand-entry-handler'
 import { denormalizePagePath } from '../../shared/lib/page-path/denormalize-page-path'
@@ -941,6 +942,10 @@ export default class HotReloader {
         edgeServerStats: () => this.edgeServerStats,
       }),
     ]
+
+    // trigger invalidation to ensure any previous callbacks
+    // are handled in the on-demand-entry-handler
+    getInvalidator()?.invalidate()
   }
 
   public async stop(): Promise<void> {

diff --git a/packages/next/server/dev/next-dev-server.ts b/packages/next/server/dev/next-dev-server.ts
@@ -252,7 +252,7 @@ export default class DevServer extends Server {
     )
 
     let resolved = false
-    return new Promise((resolve, reject) => {
+    return new Promise(async (resolve, reject) => {
       // Watchpack doesn't emit an event for an empty directory
       fs.readdir(this.pagesDir, (_, files) => {
         if (files?.length) {
@@ -265,7 +265,9 @@ export default class DevServer extends Server {
         }
       })
 
-      const wp = (this.webpackWatcher = new Watchpack())
+      const wp = (this.webpackWatcher = new Watchpack({
+        ignored: /(node_modules|\.next)/,
+      }))
       const pages = [this.pagesDir]
       const app = this.appDir ? [this.appDir] : []
       const directories = [...pages, ...app]
@@ -275,16 +277,44 @@ export default class DevServer extends Server {
       )
       let nestedMiddleware: string[] = []
 
-      wp.watch(files, directories, 0)
+      const envFiles = [
+        '.env.development.local',
+        '.env.local',
+        '.env.development',
+        '.env',
+      ].map((file) => pathJoin(this.dir, file))
+
+      files.push(...envFiles)
+      wp.watch({ directories: [this.dir], startTime: 0 })
+      const envFileTimes = new Map()
 
       wp.on('aggregated', async () => {
         let middlewareMatcher: RegExp | undefined
         const routedPages: string[] = []
         const knownFiles = wp.getTimeInfoEntries()
         const appPaths: Record<string, string> = {}
         const edgeRoutesSet = new Set<string>()
+        let envChange = false
 
         for (const [fileName, meta] of knownFiles) {
+          if (
+            !files.includes(fileName) &&
+            !directories.some((dir) => fileName.startsWith(dir))
+          ) {
+            continue
+          }
+
+          if (envFiles.includes(fileName)) {
+            if (
+              envFileTimes.get(fileName) &&
+              envFileTimes.get(fileName) !== meta.timestamp
+            ) {
+              envChange = true
+            }
+            envFileTimes.set(fileName, meta.timestamp)
+            continue
+          }
+
           if (
             meta?.accuracy === undefined ||
             !regexPageExtension.test(fileName)
@@ -358,6 +388,12 @@ export default class DevServer extends Server {
           routedPages.push(pageName)
         }
 
+        if (envChange) {
+          this.loadEnvConfig({ dev: true, forceReload: true })
+          await this.hotReloader?.stop()
+          await this.hotReloader?.start()
+        }
+
         if (nestedMiddleware.length > 0) {
           Log.error(
             new NestedMiddlewareError(nestedMiddleware, this.dir, this.pagesDir)

diff --git a/packages/next/server/dev/on-demand-entry-handler.ts b/packages/next/server/dev/on-demand-entry-handler.ts
@@ -127,6 +127,10 @@ export const entries: {
 let invalidator: Invalidator
 export const getInvalidator = () => invalidator
 
+const doneCallbacks: EventEmitter | null = new EventEmitter()
+const lastClientAccessPages = ['']
+const lastServerAccessPagesForAppDir = ['']
+
 export function onDemandEntryHandler({
   maxInactiveAge,
   multiCompiler,
@@ -145,9 +149,6 @@ export function onDemandEntryHandler({
   appDir?: string
 }) {
   invalidator = new Invalidator(multiCompiler)
-  const doneCallbacks: EventEmitter | null = new EventEmitter()
-  const lastClientAccessPages = ['']
-  const lastServerAccessPagesForAppDir = ['']
 
   const startBuilding = (_compilation: webpack.Compilation) => {
     invalidator.startBuilding()
@@ -223,11 +224,7 @@ export function onDemandEntryHandler({
   const pingIntervalTime = Math.max(1000, Math.min(5000, maxInactiveAge))
 
   setInterval(function () {
-    disposeInactiveEntries(
-      lastClientAccessPages,
-      lastServerAccessPagesForAppDir,
-      maxInactiveAge
-    )
+    disposeInactiveEntries(maxInactiveAge)
   }, pingIntervalTime + 1000).unref()
 
   function handleAppDirPing(
@@ -397,11 +394,7 @@ export function onDemandEntryHandler({
   }
 }
 
-function disposeInactiveEntries(
-  lastClientAccessPages: string[],
-  lastServerAccessPagesForAppDir: string[],
-  maxInactiveAge: number
-) {
+function disposeInactiveEntries(maxInactiveAge: number) {
   Object.keys(entries).forEach((page) => {
     const { lastActiveTime, status, dispose, bundlePath } = entries[page]
 
@@ -467,19 +460,19 @@ class Invalidator {
     this.building = true
 
     if (!keys || keys.length === 0) {
-      this.multiCompiler.compilers[0].watching.invalidate()
-      this.multiCompiler.compilers[1].watching.invalidate()
-      this.multiCompiler.compilers[2].watching.invalidate()
+      this.multiCompiler.compilers[0].watching?.invalidate()
+      this.multiCompiler.compilers[1].watching?.invalidate()
+      this.multiCompiler.compilers[2].watching?.invalidate()
       return
     }
 
     for (const key of keys) {
       if (key === 'client') {
-        this.multiCompiler.compilers[0].watching.invalidate()
+        this.multiCompiler.compilers[0].watching?.invalidate()
       } else if (key === 'server') {
-        this.multiCompiler.compilers[1].watching.invalidate()
+        this.multiCompiler.compilers[1].watching?.invalidate()
       } else if (key === 'edgeServer') {
-        this.multiCompiler.compilers[2].watching.invalidate()
+        this.multiCompiler.compilers[2].watching?.invalidate()
       }
     }
   }

diff --git a/packages/next/server/next-server.ts b/packages/next/server/next-server.ts
@@ -160,8 +160,14 @@ export default class NextNodeServer extends BaseServer {
       ? (compression() as ExpressMiddleware)
       : undefined
 
-  protected loadEnvConfig({ dev }: { dev: boolean }) {
-    loadEnvConfig(this.dir, dev, Log)
+  protected loadEnvConfig({
+    dev,
+    forceReload,
+  }: {
+    dev: boolean
+    forceReload?: boolean
+  }) {
+    loadEnvConfig(this.dir, dev, Log, forceReload)
   }
 
   protected getPublicDir(): string {

diff --git a/packages/next/types/misc.d.ts b/packages/next/types/misc.d.ts
@@ -361,6 +361,12 @@ declare module 'next/dist/compiled/watchpack' {
 
   class Watchpack extends EventEmitter {
     constructor(options?: any)
+    watch(params: {
+      files?: string[]
+      directories?: string[]
+      startTime?: number
+      missing?: string[]
+    }): void
     watch(files: string[], directories: string[], startTime?: number): void
     close(): void
 

diff --git a/test/integration/env-config/app/pages/another-global.js b/test/integration/env-config/app/pages/another-global.js
@@ -0,0 +1 @@
+export default () => <p>{process.env.NEXT_PUBLIC_HELLO_WORLD}</p>
diff --git a/test/integration/env-config/app/pages/api/all.js b/test/integration/env-config/app/pages/api/all.js
@@ -21,18 +21,22 @@ const variables = [
   'ENV_FILE_EXPANDED_CONCAT',
   'ENV_FILE_EXPANDED_ESCAPED',
   'ENV_FILE_KEY_EXCLAMATION',
+  'NEW_ENV_KEY',
+  'NEW_ENV_LOCAL_KEY',
+  'NEW_ENV_DEV_KEY',
+  'NEXT_PUBLIC_HELLO_WORLD',
 ]
 
-const items = {
-  nextConfigEnv: process.env.nextConfigEnv,
-  nextConfigPublicEnv: process.env.nextConfigPublicEnv,
-}
+export default async (req, res) => {
+  const items = {
+    nextConfigEnv: process.env.nextConfigEnv,
+    nextConfigPublicEnv: process.env.nextConfigPublicEnv,
+  }
 
-variables.forEach((variable) => {
-  items[variable] = process.env[variable]
-})
+  variables.forEach((variable) => {
+    items[variable] = process.env[variable]
+  })
 
-export default async (req, res) => {
   // Only for testing, don't do this...
   res.json(items)
 }
diff --git a/test/integration/env-config/app/pages/index.js b/test/integration/env-config/app/pages/index.js
@@ -21,6 +21,10 @@ const variables = [
   'ENV_FILE_EXPANDED_CONCAT',
   'ENV_FILE_EXPANDED_ESCAPED',
   'ENV_FILE_KEY_EXCLAMATION',
+  'NEW_ENV_KEY',
+  'NEW_ENV_LOCAL_KEY',
+  'NEW_ENV_DEV_KEY',
+  'NEXT_PUBLIC_HELLO_WORLD',
 ]
 
 export async function getStaticProps() {

diff --git a/test/integration/env-config/app/pages/some-ssg.js b/test/integration/env-config/app/pages/some-ssg.js
@@ -21,6 +21,10 @@ const variables = [
   'ENV_FILE_EXPANDED_CONCAT',
   'ENV_FILE_EXPANDED_ESCAPED',
   'ENV_FILE_KEY_EXCLAMATION',
+  'NEW_ENV_KEY',
+  'NEW_ENV_LOCAL_KEY',
+  'NEW_ENV_DEV_KEY',
+  'NEXT_PUBLIC_HELLO_WORLD',
 ]
 
 export async function getStaticProps() {

diff --git a/test/integration/env-config/app/pages/some-ssp.js b/test/integration/env-config/app/pages/some-ssp.js
@@ -21,6 +21,10 @@ const variables = [
   'ENV_FILE_EXPANDED_CONCAT',
   'ENV_FILE_EXPANDED_ESCAPED',
   'ENV_FILE_KEY_EXCLAMATION',
+  'NEW_ENV_KEY',
+  'NEW_ENV_LOCAL_KEY',
+  'NEW_ENV_DEV_KEY',
+  'NEXT_PUBLIC_HELLO_WORLD',
 ]
 
 export async function getServerSideProps() {