From 702577c851bdf54a47535a3da736016746c03dfd Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Wed, 16 Aug 2023 14:45:31 +0700
Subject: [PATCH 01/14] WIP: Get it working

---
 packages/internal/src/routes.ts               |  23 ++-
 packages/project-config/src/paths.ts          |  43 ++++-
 packages/vite/src/buildFeServer.ts            |   4 +-
 packages/vite/src/buildRscFeServer.ts         |   4 +-
 packages/vite/src/devFeServer.ts              | 133 ++++---------
 packages/vite/src/runFeServer.ts              | 177 ++----------------
 .../streaming/createReactStreamingHandler.ts  |  83 ++++++++
 packages/vite/src/streaming/streamHelpers.ts  |  21 +--
 packages/vite/src/types.ts                    |  11 +-
 9 files changed, 200 insertions(+), 299 deletions(-)
 create mode 100644 packages/vite/src/streaming/createReactStreamingHandler.ts

diff --git a/packages/internal/src/routes.ts b/packages/internal/src/routes.ts
index 21e460cd8fa6..eb202df28424 100644
--- a/packages/internal/src/routes.ts
+++ b/packages/internal/src/routes.ts
@@ -67,18 +67,23 @@ export function warningForDuplicateRoutes() {
   return message.trimEnd()
 }
 
-export interface RouteSpec {
-  name: string
-  path: string
-  hasParams: boolean
+export interface RWRouteManifestItem {
+  name: string // <-- AnalyzedRoute.name, RouteSpec.name
+  pathDefinition: string // <-- AnalyzedRoute.path, RouteSpec.path
+  matchRegexString: string | null // xAR, RouteSpec.matchRegexString
+  routeHooks: string | null // xAR, RouteSpec.routeHooks BUT in RouteSpec its the src path, here its the dist path
+  bundle: string | null // xAR, xRS
+  hasParams: boolean // xAR, RouteSpec.hasParams
+  redirect: { to: string; permanent: boolean } | null // xAR (not same type), RouteSpec.redirect
+  renderMode: 'html' | 'stream' // x, RouteSpec.renderMode
+  // Probably want isNotFound here, so we can attach a separate 404 handler
+}
+
+export interface RouteSpec extends RWRouteManifestItem {
   id: string
   isNotFound: boolean
   filePath: string | undefined
   relativeFilePath: string | undefined
-  routeHooks: string | undefined | null
-  matchRegexString: string | null
-  redirect: { to: string; permanent: boolean } | null
-  renderMode: 'stream' | 'html'
 }
 
 export const getProjectRoutes = (): RouteSpec[] => {
@@ -92,7 +97,7 @@ export const getProjectRoutes = (): RouteSpec[] => {
 
     return {
       name: route.isNotFound ? 'NotFoundPage' : route.name,
-      path: route.isNotFound ? 'notfound' : route.path,
+      pathDefinition: route.isNotFound ? 'notfound' : route.path,
       hasParams: route.hasParameters,
       id: route.id,
       isNotFound: route.isNotFound,
diff --git a/packages/project-config/src/paths.ts b/packages/project-config/src/paths.ts
index 1a38bb8daba4..84304e5ee4f4 100644
--- a/packages/project-config/src/paths.ts
+++ b/packages/project-config/src/paths.ts
@@ -262,16 +262,43 @@ export const getRouteHookForPage = (pagePath: string | undefined | null) => {
 
   // We just use fg, so if they make typos in the routeHook file name,
   // it's all good, we'll still find it
-  return fg
-    .sync('*.routeHooks.{js,ts,tsx,jsx}', {
-      absolute: true,
-      cwd: path.dirname(pagePath), // the page's folder
-    })
-    .at(0)
+  return (
+    fg
+      .sync('*.routeHooks.{js,ts,tsx,jsx}', {
+        absolute: true,
+        cwd: path.dirname(pagePath), // the page's folder
+      })
+      .at(0) || null
+  )
 }
 
-export const getAppRouteHook = () => {
-  return resolveFile(path.join(getPaths().web.src, 'App.routeHooks'))
+/**
+ * Use this function to find the app route hook.
+ * If it is present, you get the path to the file - in prod, you get the built version in dist.
+ * In dev, you get the source version.
+ *
+ * @param forProd
+ * @returns string | null
+ */
+export const getAppRouteHook = (forProd = false) => {
+  const rwPaths = getPaths()
+
+  if (forProd) {
+    const distAppRouteHook = path.join(
+      rwPaths.web.distRouteHooks,
+      'App.routeHooks.js'
+    )
+
+    try {
+      // Stat sync throws if file doesn't exist
+      fs.statSync(distAppRouteHook).isFile()
+      return distAppRouteHook
+    } catch (e) {
+      return null
+    }
+  }
+
+  return resolveFile(path.join(rwPaths.web.src, 'App.routeHooks'))
 }
 
 /**
diff --git a/packages/vite/src/buildFeServer.ts b/packages/vite/src/buildFeServer.ts
index 2cfaca286e94..9267602c1428 100644
--- a/packages/vite/src/buildFeServer.ts
+++ b/packages/vite/src/buildFeServer.ts
@@ -144,7 +144,7 @@ export const buildFeServer = async ({ verbose, webDir }: BuildOptions = {}) => {
   const routesList = getProjectRoutes()
 
   const routeManifest = routesList.reduce<RWRouteManifest>((acc, route) => {
-    acc[route.path] = {
+    acc[route.pathDefinition] = {
       name: route.name,
       bundle: route.relativeFilePath
         ? clientBuildManifest[route.relativeFilePath]?.file
@@ -152,7 +152,7 @@ export const buildFeServer = async ({ verbose, webDir }: BuildOptions = {}) => {
       matchRegexString: route.matchRegexString,
       // @NOTE this is the path definition, not the actual path
       // E.g. /blog/post/{id:Int}
-      pathDefinition: route.path,
+      pathDefinition: route.pathDefinition,
       hasParams: route.hasParams,
       routeHooks: FIXME_constructRouteHookPath(route.routeHooks),
       redirect: route.redirect
diff --git a/packages/vite/src/buildRscFeServer.ts b/packages/vite/src/buildRscFeServer.ts
index 77f26d0d0643..e618bcc4034b 100644
--- a/packages/vite/src/buildRscFeServer.ts
+++ b/packages/vite/src/buildRscFeServer.ts
@@ -218,7 +218,7 @@ export const buildRscFeServer = async ({
 
   // This is all a no-op for now
   const routeManifest = routesList.reduce<RWRouteManifest>((acc, route) => {
-    acc[route.path] = {
+    acc[route.pathDefinition] = {
       name: route.name,
       bundle: route.relativeFilePath
         ? clientBuildManifest[route.relativeFilePath].file
@@ -226,7 +226,7 @@ export const buildRscFeServer = async ({
       matchRegexString: route.matchRegexString,
       // NOTE this is the path definition, not the actual path
       // E.g. /blog/post/{id:Int}
-      pathDefinition: route.path,
+      pathDefinition: route.pathDefinition,
       hasParams: route.hasParams,
       routeHooks: null,
       redirect: route.redirect
diff --git a/packages/vite/src/devFeServer.ts b/packages/vite/src/devFeServer.ts
index 462ff2a1b9d0..ba99ae3af9ab 100644
--- a/packages/vite/src/devFeServer.ts
+++ b/packages/vite/src/devFeServer.ts
@@ -4,14 +4,11 @@ import express from 'express'
 import { createServer as createViteServer } from 'vite'
 
 import { getProjectRoutes } from '@redwoodjs/internal/dist/routes'
-import { getAppRouteHook, getConfig, getPaths } from '@redwoodjs/project-config'
-import { matchPath } from '@redwoodjs/router'
-import type { TagDescriptor } from '@redwoodjs/web'
+import { getConfig, getPaths } from '@redwoodjs/project-config'
 
+import { createReactStreamingHandler } from './streaming/createReactStreamingHandler'
 import { registerFwGlobals } from './streaming/registerGlobals'
-import { reactRenderToStream } from './streaming/streamHelpers'
-import { loadAndRunRouteHooks } from './streaming/triggerRouteHooks'
-import { ensureProcessDirWeb, stripQueryStringAndHashFromPath } from './utils'
+import { ensureProcessDirWeb } from './utils'
 
 // TODO (STREAMING) Just so it doesn't error out. Not sure how to handle this.
 globalThis.__REDWOOD__PRERENDER_PAGES = {}
@@ -47,101 +44,47 @@ async function createServer() {
   // use vite's connect instance as middleware
   app.use(vite.middlewares)
 
-  app.use('*', async (req, res, next) => {
-    const currentPathName = stripQueryStringAndHashFromPath(req.originalUrl)
-    globalThis.__REDWOOD__HELMET_CONTEXT = {}
-
-    try {
-      const routes = getProjectRoutes()
-
-      // Do a simple match with regex, don't bother parsing params yet
-      const currentRoute = routes.find((route) => {
-        if (!route.matchRegexString) {
-          // This is the 404/NotFoundPage case
-          return false
-        }
-
-        const matches = [
-          ...currentPathName.matchAll(new RegExp(route.matchRegexString, 'g')),
-        ]
-
-        return matches.length > 0
-      })
-
-      let metaTags: TagDescriptor[] = []
-
-      if (currentRoute?.redirect) {
-        return res.redirect(currentRoute.redirect.to)
-      }
-
-      if (currentRoute) {
-        const parsedParams = currentRoute.hasParams
-          ? matchPath(currentRoute.path, currentPathName).params
-          : undefined
-
-        const routeHookOutput = await loadAndRunRouteHooks({
-          paths: [getAppRouteHook(), currentRoute.routeHooks],
-          reqMeta: {
-            req,
-            parsedParams,
-          },
-          viteDevServer: vite, // because its dev
-        })
-
-        metaTags = routeHookOutput.meta
-      }
-
-      if (!currentRoute) {
-        // TODO (STREAMING) do something
-      }
-
-      if (!rwPaths.web.entryServer || !rwPaths.web.entryClient) {
-        throw new Error(
-          'Vite entry points not found. Please check that your project has ' +
-            'an entry.client.{jsx,tsx} and entry.server.{jsx,tsx} file in ' +
-            'the web/src directory.'
-        )
-      }
-
-      // 3. Load the server entry. vite.ssrLoadModule automatically transforms
-      //    your ESM source code to be usable in Node.js! There is no bundling
-      //    required, and provides efficient invalidation similar to HMR.
-      const { ServerEntry } = await vite.ssrLoadModule(rwPaths.web.entryServer)
-
-      const pageWithJs = currentRoute?.renderMode !== 'html'
-
-      res.setHeader('content-type', 'text/html; charset=utf-8')
-
-      reactRenderToStream({
-        ServerEntry,
-        currentPathName,
-        metaTags,
-        includeJs: pageWithJs,
-        res,
-      })
-    } catch (e) {
-      // TODO (STREAMING) Is this what we want to do?
-      // send back a SPA page
-      // res.status(200).set({ 'Content-Type': 'text/html' }).end(template)
-
-      // If an error is caught, let Vite fix the stack trace so it maps back to
-      // your actual source code.
-      vite.ssrFixStacktrace(e as any)
-      next(e)
+  const routes = getProjectRoutes()
+
+  // TODO (STREAMING) CSS is handled by Vite in dev mode, we don't need to
+  // worry about it in dev but..... it causes a flash of unstyled content.
+  // For now I'm just injecting index css here
+  // Look at collectStyles in packages/vite/src/fully-react/find-styles.ts
+  const FIXME_HardcodedIndexCss = ['index.css']
+
+  const x = routes.map(async (route) => {
+    console.log(`Attaching handler for ${route.name}`)
+    const routeHandler = await createReactStreamingHandler(
+      route,
+      FIXME_HardcodedIndexCss,
+      vite
+    )
+
+    // if it is a 404
+    if (!route.matchRegexString) {
+      return
     }
+
+    const expressPathDef = route.hasParams
+      ? route.matchRegexString
+      : route.pathDefinition
+
+    app.get(expressPathDef, routeHandler)
   })
 
+  await Promise.all(x)
+
   const port = getConfig().web.port
   console.log(`Started server on http://localhost:${port}`)
   return await app.listen(port)
 }
 
-let devApp = createServer()
+createServer()
 
-process.stdin.on('data', async (data) => {
-  const str = data.toString().trim().toLowerCase()
-  if (str === 'rs' || str === 'restart') {
-    ;(await devApp).close()
-    devApp = createServer()
-  }
-})
+// process.stdin.on('data', async (data) => {
+//   const str = data.toString().trim().toLowerCase()
+//   if (str === 'rs' || str === 'restart') {
+//     ;(await devApp).close()
+//     devApp = createServer()
+//   }
+// })
diff --git a/packages/vite/src/runFeServer.ts b/packages/vite/src/runFeServer.ts
index 3170de8eb25b..4b6d32840a74 100644
--- a/packages/vite/src/runFeServer.ts
+++ b/packages/vite/src/runFeServer.ts
@@ -11,17 +11,13 @@ import { config as loadDotEnv } from 'dotenv-defaults'
 import express from 'express'
 import { createProxyMiddleware } from 'http-proxy-middleware'
 import isbot from 'isbot'
-import { renderToPipeableStream } from 'react-dom/server'
 import type { Manifest as ViteBuildManifest } from 'vite'
 
 import { getConfig, getPaths } from '@redwoodjs/project-config'
-import { matchPath } from '@redwoodjs/router'
-import type { TagDescriptor } from '@redwoodjs/web'
 
+import { createReactStreamingHandler } from './streaming/createReactStreamingHandler'
 import { registerFwGlobals } from './streaming/registerGlobals'
-import { loadAndRunRouteHooks } from './streaming/triggerRouteHooks'
 import { RWRouteManifest } from './types'
-import { stripQueryStringAndHashFromPath } from './utils'
 
 /**
  * TODO (STREAMING)
@@ -105,169 +101,26 @@ export async function runFeServer() {
     })
   )
 
-  // 👉 3. Handle all other requests with the server entry
-  // This is where we match the url to the correct route, and render it
-  // We also call the relevant routeHooks here
-  app.use('*', async (req, res) => {
-    const currentPathName = stripQueryStringAndHashFromPath(req.originalUrl)
+  const collectedCss = indexEntry.css || []
 
-    try {
-      const { ServerEntry } = await import(rwPaths.web.distEntryServer)
+  const x = Object.values(routeManifest).map(async (route) => {
+    console.log(`Attaching handler for ${route.name}`)
+    const routeHandler = await createReactStreamingHandler(route, collectedCss)
 
-      // TODO (STREAMING) should we generate individual express Routes for each Route?
-      // This would make handling 404s and favicons / public assets etc. easier
-      const currentRoute = Object.values(routeManifest).find((route) => {
-        if (!route.matchRegexString) {
-          // This is the 404/NotFoundPage case
-          return false
-        }
-
-        const matches = [
-          ...currentPathName.matchAll(new RegExp(route.matchRegexString, 'g')),
-        ]
-        return matches.length > 0
-      })
-
-      // Doesn't match any of the defined Routes
-      // Render 404 page, and send back 404 status
-      if (!currentRoute) {
-        // TODO (STREAMING) should we CONST it?
-        const fourOhFourRoute = routeManifest['notfound']
-
-        if (!fourOhFourRoute) {
-          return res.sendStatus(404)
-        }
-
-        const assetMap = JSON.stringify({ css: indexEntry.css })
-
-        const { pipe } = renderToPipeableStream(
-          ServerEntry({
-            url: currentPathName,
-            routeContext: null,
-            css: indexEntry.css,
-          }),
-          {
-            bootstrapScriptContent: `window.__assetMap = function() { return ${assetMap} }`,
-            // @NOTE have to add slash so subpaths still pick up the right file
-            // Vite is currently producing modules not scripts: https://vitejs.dev/config/build-options.html#build-target
-            bootstrapModules: [
-              '/' + indexEntry.file,
-              '/' + fourOhFourRoute.bundle,
-            ],
-            onShellReady() {
-              res.setHeader('content-type', 'text/html')
-              res.status(404)
-              pipe(res)
-            },
-          }
-        )
-
-        return
-      }
-
-      let metaTags: TagDescriptor[] = []
-
-      if (currentRoute?.redirect) {
-        // TODO (STREAMING) deal with permanent/temp
-        // Short-circuit, and return a 301 or 302
-        return res.redirect(currentRoute.redirect.to)
-      }
-
-      if (currentRoute) {
-        // TODO (STREAMING) hardcoded JS file, watchout if we switch to ESM!
-        const appRouteHooksPath = path.join(
-          rwPaths.web.distRouteHooks,
-          'App.routeHooks.js'
-        )
-
-        let appRouteHooksExists = false
-        try {
-          appRouteHooksExists = (await fs.stat(appRouteHooksPath)).isFile()
-        } catch {
-          // noop
-        }
-
-        // Make sure we access the dist routeHooks!
-        const routeHookPaths = [
-          appRouteHooksExists ? appRouteHooksPath : null,
-          currentRoute.routeHooks
-            ? path.join(rwPaths.web.distRouteHooks, currentRoute.routeHooks)
-            : null,
-        ]
-
-        const parsedParams = currentRoute.hasParams
-          ? matchPath(currentRoute.pathDefinition, currentPathName).params
-          : undefined
-
-        const routeHookOutput = await loadAndRunRouteHooks({
-          paths: routeHookPaths,
-          reqMeta: {
-            req,
-            parsedParams,
-          },
-        })
-
-        metaTags = routeHookOutput.meta
-      }
-
-      const pageWithJs = currentRoute.renderMode !== 'html'
-      // @NOTE have to add slash so subpaths still pick up the right file
-      const bootstrapModules = pageWithJs
-        ? ([
-            '/' + indexEntry.file,
-            currentRoute.bundle && '/' + currentRoute.bundle,
-          ].filter(Boolean) as string[])
-        : undefined
-
-      const isSeoCrawler = checkUaForSeoCrawler(req.get('user-agent'))
-
-      const { pipe, abort } = renderToPipeableStream(
-        ServerEntry({
-          url: currentPathName,
-          css: indexEntry.css,
-          meta: metaTags,
-        }),
-        {
-          bootstrapScriptContent: pageWithJs
-            ? `window.__assetMap = function() { return ${JSON.stringify({
-                css: indexEntry.css,
-                meta: metaTags,
-              })} }`
-            : undefined,
-          bootstrapModules,
-          onShellReady() {
-            if (!isSeoCrawler) {
-              res.setHeader('content-type', 'text/html; charset=utf-8')
-              pipe(res)
-            }
-          },
-          onAllReady() {
-            if (isSeoCrawler) {
-              res.setHeader('content-type', 'text/html; charset=utf-8')
-              pipe(res)
-            }
-          },
-          onError(error) {
-            console.error(error)
-          },
-        }
-      )
-
-      // TODO (STREAMING) make the timeout configurable
-      setTimeout(() => {
-        abort()
-      }, 10_000)
-    } catch (e) {
-      console.error(e)
-
-      // streaming no longer requires us to send back a blank page
-      // React will automatically switch to client rendering on error
-      return res.sendStatus(500)
+    // if it is a 404
+    if (!route.matchRegexString) {
+      return
     }
 
-    return
+    const expressPathDef = route.hasParams
+      ? route.matchRegexString
+      : route.pathDefinition
+
+    app.get(expressPathDef, routeHandler)
   })
 
+  await Promise.all(x)
+
   app.listen(rwConfig.web.port)
   console.log(
     `Started production FE server on http://localhost:${rwConfig.web.port}`
diff --git a/packages/vite/src/streaming/createReactStreamingHandler.ts b/packages/vite/src/streaming/createReactStreamingHandler.ts
new file mode 100644
index 000000000000..10f58a8ff872
--- /dev/null
+++ b/packages/vite/src/streaming/createReactStreamingHandler.ts
@@ -0,0 +1,83 @@
+import path from 'path'
+
+import type { Request, Response } from 'express'
+import type { ViteDevServer } from 'vite'
+
+import type { RWRouteManifestItem } from '@redwoodjs/internal'
+import { getAppRouteHook, getPaths } from '@redwoodjs/project-config'
+import type { TagDescriptor } from '@redwoodjs/web'
+
+// import { stripQueryStringAndHashFromPath } from '../utils'
+
+import { reactRenderToStream } from './streamHelpers'
+import { loadAndRunRouteHooks } from './triggerRouteHooks'
+
+export const createReactStreamingHandler = async (
+  { redirect, routeHooks }: RWRouteManifestItem,
+  cssLinks: string[], // this is different between prod and dev, so we pass it in
+  viteDevServer?: ViteDevServer
+) => {
+  const rwPaths = getPaths()
+
+  const isProd = !viteDevServer
+
+  let entryServerImport: any
+
+  if (viteDevServer) {
+    if (!rwPaths.web.entryServer || !rwPaths.web.entryClient) {
+      throw new Error(
+        'Vite entry points not found. Please check that your project has ' +
+          'an entry.client.{jsx,tsx} and entry.server.{jsx,tsx} file in ' +
+          'the web/src directory.'
+      )
+    }
+
+    entryServerImport = await viteDevServer.ssrLoadModule(
+      rwPaths.web.entryServer
+    )
+  } else {
+    entryServerImport = await import(rwPaths.web.distEntryServer)
+  }
+
+  const ServerEntry = entryServerImport.ServerEntry || entryServerImport.default
+
+  return async (req: Request, res: Response) => {
+    if (redirect) {
+      res.redirect(redirect.to)
+    }
+
+    const currentPathName = req.path
+
+    let metaTags: TagDescriptor[] = []
+
+    let routeHookPath = routeHooks
+
+    if (isProd) {
+      routeHookPath = routeHooks
+        ? path.join(rwPaths.web.distRouteHooks, routeHooks)
+        : null
+    }
+
+    // @TODO can we load the route hook outside the handler?
+    const routeHookOutput = await loadAndRunRouteHooks({
+      paths: [getAppRouteHook(isProd), routeHookPath],
+      reqMeta: {
+        req,
+        parsedParams: req.params,
+      },
+      viteDevServer,
+    })
+
+    metaTags = routeHookOutput.meta
+
+    reactRenderToStream({
+      ServerEntry,
+      currentPathName,
+      metaTags,
+      cssLinks,
+      includeJs: true,
+      isProd,
+      res,
+    })
+  }
+}
diff --git a/packages/vite/src/streaming/streamHelpers.ts b/packages/vite/src/streaming/streamHelpers.ts
index 4b57a24c0d33..acd97ee5307a 100644
--- a/packages/vite/src/streaming/streamHelpers.ts
+++ b/packages/vite/src/streaming/streamHelpers.ts
@@ -19,7 +19,9 @@ interface RenderToStreamArgs {
   ServerEntry: any
   currentPathName: string
   metaTags: TagDescriptor[]
+  cssLinks: string[]
   includeJs: boolean
+  isProd: boolean
   res: Writable
 }
 
@@ -27,28 +29,24 @@ export function reactRenderToStream({
   ServerEntry,
   currentPathName,
   metaTags,
+  cssLinks,
   includeJs,
+  isProd,
   res,
 }: RenderToStreamArgs) {
   const rwPaths = getPaths()
 
-  const bootstrapModules = [
-    path.join(__dirname, '../../inject', 'reactRefresh.js'),
-  ]
+  const bootstrapModules = isProd
+    ? []
+    : [path.join(__dirname, '../../inject', 'reactRefresh.js')]
 
   if (includeJs) {
     // type casting: guaranteed to have entryClient by this stage, because checks run earlier
     bootstrapModules.push(rwPaths.web.entryClient as string)
   }
 
-  // TODO (STREAMING) CSS is handled by Vite in dev mode, we don't need to
-  // worry about it in dev but..... it causes a flash of unstyled content.
-  // For now I'm just injecting index css here
-  // Looks at collectStyles in packages/vite/src/fully-react/find-styles.ts
-  const FIXME_HardcodedIndexCss = ['index.css']
-
   const assetMap = JSON.stringify({
-    css: FIXME_HardcodedIndexCss,
+    css: cssLinks,
     meta: metaTags,
   })
 
@@ -69,7 +67,7 @@ export function reactRenderToStream({
       },
       ServerEntry({
         url: currentPathName,
-        css: FIXME_HardcodedIndexCss,
+        css: cssLinks,
         meta: metaTags,
       })
     ),
@@ -86,6 +84,7 @@ export function reactRenderToStream({
     }
   )
 }
+
 function createServerInjectionStream({
   outputStream,
   injectionState,
diff --git a/packages/vite/src/types.ts b/packages/vite/src/types.ts
index 445f3226a572..8d29722d3863 100644
--- a/packages/vite/src/types.ts
+++ b/packages/vite/src/types.ts
@@ -9,16 +9,7 @@
  *
  * **All** of these properties are used by the prod FE server
  */
-export interface RWRouteManifestItem {
-  matchRegexString: string | null // xAR, RouteSpec.matchRegexString
-  routeHooks: string | null // xAR, RouteSpec.routeHooks BUT in RouteSpec its the src path, here its the dist path
-  bundle: string | null // xAR, xRS
-  pathDefinition: PathDefinition // <-- AnalyzedRoute.path, RouteSpec.path
-  hasParams: boolean // xAR, RouteSpec.hasParams
-  name: string // <-- AnalyzedRoute.name, RouteSpec.name
-  redirect: { to: string; permanent: boolean } | null // xAR (not same type), RouteSpec.redirect
-  renderMode: 'html' | 'stream' // x, RouteSpec.renderMode
-}
+import type { RWRouteManifestItem } from '@redwoodjs/internal'
 
 export type RWRouteManifest = Record<PathDefinition, RWRouteManifestItem>
 

From 8b1278e66a0f62db5bfb2ac9be2063b87594a6dc Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Wed, 16 Aug 2023 14:48:42 +0700
Subject: [PATCH 02/14] Use web server proxy

---
 .../vite/src/streaming/registerGlobals.ts     | 23 ++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/packages/vite/src/streaming/registerGlobals.ts b/packages/vite/src/streaming/registerGlobals.ts
index 467920923251..2c9143e6bc46 100644
--- a/packages/vite/src/streaming/registerGlobals.ts
+++ b/packages/vite/src/streaming/registerGlobals.ts
@@ -33,9 +33,26 @@ export const registerFwGlobals = () => {
       if (/^[a-zA-Z][a-zA-Z\d+\-.]*?:/.test(apiPath)) {
         return apiPath
       } else {
-        return (
-          'http://' + rwConfig.api.host + ':' + rwConfig.api.port + '/graphql'
-        )
+        if (
+          process.env.NODE_ENV === 'production' &&
+          rwConfig.api.host.includes('localhost')
+        ) {
+          console.log('------------------ WARNING ! -------------------------')
+          console.warn()
+          console.warn()
+
+          console.warn(
+            `Your api host is ${rwConfig.api.host}. Localhost is unlikely to work in production`
+          )
+
+          console.warn()
+          console.warn()
+
+          console.log('------------------ WARNING ! -------------------------')
+        }
+        // @TODO (Streaming): Temporarily give it the web server's proxy.
+        // I need to think about how best to address this.
+        return 'http://' + rwConfig.web.host + ':' + rwConfig.web.port + apiPath
       }
     })(),
   }

From 9da1e2f106ae66095688f2317def2408bc8336aa Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Wed, 16 Aug 2023 16:41:21 +0700
Subject: [PATCH 03/14] Allow overriding ssr graphql URL

---
 .../vite/src/streaming/registerGlobals.ts     | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/packages/vite/src/streaming/registerGlobals.ts b/packages/vite/src/streaming/registerGlobals.ts
index 2c9143e6bc46..46e0eec6e051 100644
--- a/packages/vite/src/streaming/registerGlobals.ts
+++ b/packages/vite/src/streaming/registerGlobals.ts
@@ -35,24 +35,33 @@ export const registerFwGlobals = () => {
       } else {
         if (
           process.env.NODE_ENV === 'production' &&
-          rwConfig.api.host.includes('localhost')
+          !process.env.RWJS_EXP_SSR_GRAPHQL_ENDPOINT?.length
         ) {
+          const proxiedApiUrl =
+            'http://' + rwConfig.web.host + ':' + rwConfig.web.port + apiPath
+
           console.log('------------------ WARNING ! -------------------------')
           console.warn()
           console.warn()
 
           console.warn(
-            `Your api host is ${rwConfig.api.host}. Localhost is unlikely to work in production`
+            `You haven't configured your API absolute url. Localhost is unlikely to work in production`
           )
 
+          console.warn(`Using ${proxiedApiUrl}`)
           console.warn()
+
+          console.warn(
+            'You can override this for by setting RWJS_EXP_SSR_GRAPHQL_ENDPOINT in your environment vars'
+          )
           console.warn()
 
           console.log('------------------ WARNING ! -------------------------')
+
+          return proxiedApiUrl
         }
-        // @TODO (Streaming): Temporarily give it the web server's proxy.
-        // I need to think about how best to address this.
-        return 'http://' + rwConfig.web.host + ':' + rwConfig.web.port + apiPath
+
+        return process.env.RWJS_EXP_SSR_GRAPHQL_ENDPOINT as string
       }
     })(),
   }

From 658158730dcb0a8e0a656b0fd8379edd428d3798 Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Wed, 16 Aug 2023 16:46:22 +0700
Subject: [PATCH 04/14] Fix bundle injection

---
 packages/vite/src/devFeServer.ts              | 11 ++++++++
 packages/vite/src/runFeServer.ts              |  9 +++++--
 .../streaming/createReactStreamingHandler.ts  | 20 +++++++-------
 packages/vite/src/streaming/streamHelpers.ts  | 27 ++++++++-----------
 4 files changed, 38 insertions(+), 29 deletions(-)

diff --git a/packages/vite/src/devFeServer.ts b/packages/vite/src/devFeServer.ts
index ba99ae3af9ab..0f08d1c4be5e 100644
--- a/packages/vite/src/devFeServer.ts
+++ b/packages/vite/src/devFeServer.ts
@@ -21,14 +21,24 @@ async function createServer() {
   const app = express()
   const rwPaths = getPaths()
 
+  // ~~~ Dev time validations ~~~~
   // TODO (STREAMING) When Streaming is released Vite will be the only bundler,
   // and this file should always exist. So the error message needs to change
   // (or be removed perhaps)
+  if (!rwPaths.web.entryServer || !rwPaths.web.entryClient) {
+    throw new Error(
+      'Vite entry points not found. Please check that your project has ' +
+        'an entry.client.{jsx,tsx} and entry.server.{jsx,tsx} file in ' +
+        'the web/src directory.'
+    )
+  }
+
   if (!rwPaths.web.viteConfig) {
     throw new Error(
       'Vite config not found. You need to setup your project with Vite using `yarn rw setup vite`'
     )
   }
+  // ~~~~ Dev time validations ~~~~
 
   // Create Vite server in middleware mode and configure the app type as
   // 'custom', disabling Vite's own HTML serving logic so parent server
@@ -56,6 +66,7 @@ async function createServer() {
     console.log(`Attaching handler for ${route.name}`)
     const routeHandler = await createReactStreamingHandler(
       route,
+      rwPaths.web.entryClient as string,
       FIXME_HardcodedIndexCss,
       vite
     )
diff --git a/packages/vite/src/runFeServer.ts b/packages/vite/src/runFeServer.ts
index 4b6d32840a74..7df610ee2a44 100644
--- a/packages/vite/src/runFeServer.ts
+++ b/packages/vite/src/runFeServer.ts
@@ -80,7 +80,7 @@ export async function runFeServer() {
 
   // 👉 1. Use static handler for assets
   // For CF workers, we'd need an equivalent of this
-  app.use('/assets', express.static(rwPaths.web.dist + '/assets'))
+  app.use('/', express.static(rwPaths.web.dist, { index: false }))
 
   // 👉 2. Proxy the api server
   // TODO (STREAMING) we need to be able to specify whether proxying is required or not
@@ -102,10 +102,15 @@ export async function runFeServer() {
   )
 
   const collectedCss = indexEntry.css || []
+  const clientEntry = '/' + indexEntry.file
 
   const x = Object.values(routeManifest).map(async (route) => {
     console.log(`Attaching handler for ${route.name}`)
-    const routeHandler = await createReactStreamingHandler(route, collectedCss)
+    const routeHandler = await createReactStreamingHandler(
+      route,
+      clientEntry,
+      collectedCss
+    )
 
     // if it is a 404
     if (!route.matchRegexString) {
diff --git a/packages/vite/src/streaming/createReactStreamingHandler.ts b/packages/vite/src/streaming/createReactStreamingHandler.ts
index 10f58a8ff872..e7bb96bcb3ba 100644
--- a/packages/vite/src/streaming/createReactStreamingHandler.ts
+++ b/packages/vite/src/streaming/createReactStreamingHandler.ts
@@ -13,7 +13,8 @@ import { reactRenderToStream } from './streamHelpers'
 import { loadAndRunRouteHooks } from './triggerRouteHooks'
 
 export const createReactStreamingHandler = async (
-  { redirect, routeHooks }: RWRouteManifestItem,
+  { redirect, routeHooks, bundle }: RWRouteManifestItem,
+  clientEntryPath: string,
   cssLinks: string[], // this is different between prod and dev, so we pass it in
   viteDevServer?: ViteDevServer
 ) => {
@@ -24,16 +25,8 @@ export const createReactStreamingHandler = async (
   let entryServerImport: any
 
   if (viteDevServer) {
-    if (!rwPaths.web.entryServer || !rwPaths.web.entryClient) {
-      throw new Error(
-        'Vite entry points not found. Please check that your project has ' +
-          'an entry.client.{jsx,tsx} and entry.server.{jsx,tsx} file in ' +
-          'the web/src directory.'
-      )
-    }
-
     entryServerImport = await viteDevServer.ssrLoadModule(
-      rwPaths.web.entryServer
+      rwPaths.web.entryServer as string // already validated in dev server
     )
   } else {
     entryServerImport = await import(rwPaths.web.distEntryServer)
@@ -70,13 +63,18 @@ export const createReactStreamingHandler = async (
 
     metaTags = routeHookOutput.meta
 
+    const jsBundles = [
+      clientEntryPath, // @NOTE: must have slash in front
+      bundle && '/' + bundle,
+    ].filter(Boolean) as string[]
+
     reactRenderToStream({
       ServerEntry,
       currentPathName,
       metaTags,
       cssLinks,
-      includeJs: true,
       isProd,
+      jsBundles,
       res,
     })
   }
diff --git a/packages/vite/src/streaming/streamHelpers.ts b/packages/vite/src/streaming/streamHelpers.ts
index acd97ee5307a..2c6a1bd319e1 100644
--- a/packages/vite/src/streaming/streamHelpers.ts
+++ b/packages/vite/src/streaming/streamHelpers.ts
@@ -5,7 +5,6 @@ import React from 'react'
 
 import { renderToPipeableStream, renderToString } from 'react-dom/server'
 
-import { getPaths } from '@redwoodjs/project-config'
 import type { TagDescriptor } from '@redwoodjs/web'
 // @TODO (ESM), use exports field. Cannot import from web because of index exports
 import {
@@ -20,8 +19,8 @@ interface RenderToStreamArgs {
   currentPathName: string
   metaTags: TagDescriptor[]
   cssLinks: string[]
-  includeJs: boolean
   isProd: boolean
+  jsBundles?: string[]
   res: Writable
 }
 
@@ -30,19 +29,13 @@ export function reactRenderToStream({
   currentPathName,
   metaTags,
   cssLinks,
-  includeJs,
   isProd,
+  jsBundles = [],
   res,
 }: RenderToStreamArgs) {
-  const rwPaths = getPaths()
-
-  const bootstrapModules = isProd
-    ? []
-    : [path.join(__dirname, '../../inject', 'reactRefresh.js')]
-
-  if (includeJs) {
-    // type casting: guaranteed to have entryClient by this stage, because checks run earlier
-    bootstrapModules.push(rwPaths.web.entryClient as string)
+  if (!isProd) {
+    // For development, we need to inject the react-refresh runtime
+    jsBundles.push(path.join(__dirname, '../../inject', 'reactRefresh.js'))
   }
 
   const assetMap = JSON.stringify({
@@ -72,10 +65,12 @@ export function reactRenderToStream({
       })
     ),
     {
-      bootstrapScriptContent: includeJs
-        ? `window.__REDWOOD__ASSET_MAP = ${assetMap}`
-        : undefined,
-      bootstrapModules,
+      bootstrapScriptContent:
+        // Only insert assetMap if clientside JS will be loaded
+        jsBundles.length > 0
+          ? `window.__REDWOOD__ASSET_MAP = ${assetMap}`
+          : undefined,
+      bootstrapModules: jsBundles,
       onShellReady() {
         // Pass the react "input" stream to the injection stream
         // This intermediate stream will interweave the injected html into the react stream's <head>

From 8803a4524469f1c2c132d4b3eb6037721bb792c8 Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Wed, 16 Aug 2023 20:54:24 +0700
Subject: [PATCH 05/14] Fix bug in SSR url

---
 packages/vite/src/streaming/registerGlobals.ts | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/vite/src/streaming/registerGlobals.ts b/packages/vite/src/streaming/registerGlobals.ts
index 46e0eec6e051..b1c87b51480a 100644
--- a/packages/vite/src/streaming/registerGlobals.ts
+++ b/packages/vite/src/streaming/registerGlobals.ts
@@ -33,13 +33,13 @@ export const registerFwGlobals = () => {
       if (/^[a-zA-Z][a-zA-Z\d+\-.]*?:/.test(apiPath)) {
         return apiPath
       } else {
+        const proxiedApiUrl =
+          'http://' + rwConfig.web.host + ':' + rwConfig.web.port + apiPath
+
         if (
           process.env.NODE_ENV === 'production' &&
           !process.env.RWJS_EXP_SSR_GRAPHQL_ENDPOINT?.length
         ) {
-          const proxiedApiUrl =
-            'http://' + rwConfig.web.host + ':' + rwConfig.web.port + apiPath
-
           console.log('------------------ WARNING ! -------------------------')
           console.warn()
           console.warn()
@@ -61,7 +61,9 @@ export const registerFwGlobals = () => {
           return proxiedApiUrl
         }
 
-        return process.env.RWJS_EXP_SSR_GRAPHQL_ENDPOINT as string
+        return (
+          proxiedApiUrl || (process.env.RWJS_EXP_SSR_GRAPHQL_ENDPOINT as string)
+        )
       }
     })(),
   }

From cffb62e948d9087798cb12aa801b0e42b77e4f2a Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Wed, 16 Aug 2023 20:54:42 +0700
Subject: [PATCH 06/14] Wait for all ready when isBot

---
 packages/vite/src/devFeServer.ts              | 19 +++++----
 packages/vite/src/runFeServer.ts              | 23 ++++-------
 .../streaming/createReactStreamingHandler.ts  | 40 +++++++++++++------
 packages/vite/src/streaming/streamHelpers.ts  | 38 +++++++++++++-----
 4 files changed, 73 insertions(+), 47 deletions(-)

diff --git a/packages/vite/src/devFeServer.ts b/packages/vite/src/devFeServer.ts
index 0f08d1c4be5e..fa12d0c62345 100644
--- a/packages/vite/src/devFeServer.ts
+++ b/packages/vite/src/devFeServer.ts
@@ -62,18 +62,19 @@ async function createServer() {
   // Look at collectStyles in packages/vite/src/fully-react/find-styles.ts
   const FIXME_HardcodedIndexCss = ['index.css']
 
-  const x = routes.map(async (route) => {
-    console.log(`Attaching handler for ${route.name}`)
+  for (const route of routes) {
     const routeHandler = await createReactStreamingHandler(
-      route,
-      rwPaths.web.entryClient as string,
-      FIXME_HardcodedIndexCss,
+      {
+        route,
+        clientEntryPath: rwPaths.web.entryClient as string,
+        cssLinks: FIXME_HardcodedIndexCss,
+      },
       vite
     )
 
-    // if it is a 404
+    // @TODO if it is a 404, hand over to 404 handler
     if (!route.matchRegexString) {
-      return
+      continue
     }
 
     const expressPathDef = route.hasParams
@@ -81,9 +82,7 @@ async function createServer() {
       : route.pathDefinition
 
     app.get(expressPathDef, routeHandler)
-  })
-
-  await Promise.all(x)
+  }
 
   const port = getConfig().web.port
   console.log(`Started server on http://localhost:${port}`)
diff --git a/packages/vite/src/runFeServer.ts b/packages/vite/src/runFeServer.ts
index 7df610ee2a44..3954cb5e6f9d 100644
--- a/packages/vite/src/runFeServer.ts
+++ b/packages/vite/src/runFeServer.ts
@@ -10,7 +10,6 @@ import path from 'path'
 import { config as loadDotEnv } from 'dotenv-defaults'
 import express from 'express'
 import { createProxyMiddleware } from 'http-proxy-middleware'
-import isbot from 'isbot'
 import type { Manifest as ViteBuildManifest } from 'vite'
 
 import { getConfig, getPaths } from '@redwoodjs/project-config'
@@ -37,9 +36,6 @@ loadDotEnv({
 })
 //------------------------------------------------
 
-const checkUaForSeoCrawler = isbot.spawn()
-checkUaForSeoCrawler.exclude(['chrome-lighthouse'])
-
 export async function runFeServer() {
   const app = express()
   const rwPaths = getPaths()
@@ -104,17 +100,16 @@ export async function runFeServer() {
   const collectedCss = indexEntry.css || []
   const clientEntry = '/' + indexEntry.file
 
-  const x = Object.values(routeManifest).map(async (route) => {
-    console.log(`Attaching handler for ${route.name}`)
-    const routeHandler = await createReactStreamingHandler(
+  for (const route of Object.values(routeManifest)) {
+    const routeHandler = await createReactStreamingHandler({
       route,
-      clientEntry,
-      collectedCss
-    )
+      clientEntryPath: clientEntry,
+      cssLinks: collectedCss,
+    })
 
-    // if it is a 404
+    // if it is a 404, register it at the end somehow.
     if (!route.matchRegexString) {
-      return
+      continue
     }
 
     const expressPathDef = route.hasParams
@@ -122,9 +117,7 @@ export async function runFeServer() {
       : route.pathDefinition
 
     app.get(expressPathDef, routeHandler)
-  })
-
-  await Promise.all(x)
+  }
 
   app.listen(rwConfig.web.port)
   console.log(
diff --git a/packages/vite/src/streaming/createReactStreamingHandler.ts b/packages/vite/src/streaming/createReactStreamingHandler.ts
index e7bb96bcb3ba..2e134ef114ce 100644
--- a/packages/vite/src/streaming/createReactStreamingHandler.ts
+++ b/packages/vite/src/streaming/createReactStreamingHandler.ts
@@ -1,6 +1,7 @@
 import path from 'path'
 
 import type { Request, Response } from 'express'
+import isbot from 'isbot'
 import type { ViteDevServer } from 'vite'
 
 import type { RWRouteManifestItem } from '@redwoodjs/internal'
@@ -12,12 +13,20 @@ import type { TagDescriptor } from '@redwoodjs/web'
 import { reactRenderToStream } from './streamHelpers'
 import { loadAndRunRouteHooks } from './triggerRouteHooks'
 
+interface CreateReactStreamingHandlerOptions {
+  route: RWRouteManifestItem
+  clientEntryPath: string
+  cssLinks: string[]
+}
+
+const checkUaForSeoCrawler = isbot.spawn()
+checkUaForSeoCrawler.exclude(['chrome-lighthouse'])
+
 export const createReactStreamingHandler = async (
-  { redirect, routeHooks, bundle }: RWRouteManifestItem,
-  clientEntryPath: string,
-  cssLinks: string[], // this is different between prod and dev, so we pass it in
+  { route, clientEntryPath, cssLinks }: CreateReactStreamingHandlerOptions,
   viteDevServer?: ViteDevServer
 ) => {
+  const { redirect, routeHooks, bundle } = route
   const rwPaths = getPaths()
 
   const isProd = !viteDevServer
@@ -68,14 +77,21 @@ export const createReactStreamingHandler = async (
       bundle && '/' + bundle,
     ].filter(Boolean) as string[]
 
-    reactRenderToStream({
-      ServerEntry,
-      currentPathName,
-      metaTags,
-      cssLinks,
-      isProd,
-      jsBundles,
-      res,
-    })
+    const isSeoCrawler = checkUaForSeoCrawler(req.headers['user-agent'] || '')
+
+    reactRenderToStream(
+      {
+        ServerEntry,
+        currentPathName,
+        metaTags,
+        cssLinks,
+        isProd,
+        jsBundles,
+        res,
+      },
+      {
+        waitForAllReady: isSeoCrawler,
+      }
+    )
   }
 }
diff --git a/packages/vite/src/streaming/streamHelpers.ts b/packages/vite/src/streaming/streamHelpers.ts
index 2c6a1bd319e1..a51ccf727daf 100644
--- a/packages/vite/src/streaming/streamHelpers.ts
+++ b/packages/vite/src/streaming/streamHelpers.ts
@@ -24,15 +24,25 @@ interface RenderToStreamArgs {
   res: Writable
 }
 
-export function reactRenderToStream({
-  ServerEntry,
-  currentPathName,
-  metaTags,
-  cssLinks,
-  isProd,
-  jsBundles = [],
-  res,
-}: RenderToStreamArgs) {
+interface StreamOptions {
+  waitForAllReady?: boolean
+}
+
+export function reactRenderToStream(
+  renderOptions: RenderToStreamArgs,
+  streamOptions: StreamOptions
+) {
+  const { waitForAllReady = false } = streamOptions
+  const {
+    ServerEntry,
+    currentPathName,
+    metaTags,
+    cssLinks,
+    isProd,
+    jsBundles = [],
+    res,
+  } = renderOptions
+
   if (!isProd) {
     // For development, we need to inject the react-refresh runtime
     jsBundles.push(path.join(__dirname, '../../inject', 'reactRefresh.js'))
@@ -74,7 +84,15 @@ export function reactRenderToStream({
       onShellReady() {
         // Pass the react "input" stream to the injection stream
         // This intermediate stream will interweave the injected html into the react stream's <head>
-        pipe(intermediateStream)
+
+        if (!waitForAllReady) {
+          pipe(intermediateStream)
+        }
+      },
+      onAllReady() {
+        if (waitForAllReady) {
+          pipe(intermediateStream)
+        }
       },
     }
   )

From 1198f76953f5fa55667a1e99935e559b4e10a0d1 Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Wed, 16 Aug 2023 20:55:00 +0700
Subject: [PATCH 07/14] Save todo

---
 packages/vite/src/streaming/Streaming.todo | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 packages/vite/src/streaming/Streaming.todo

diff --git a/packages/vite/src/streaming/Streaming.todo b/packages/vite/src/streaming/Streaming.todo
new file mode 100644
index 000000000000..89421ea54f82
--- /dev/null
+++ b/packages/vite/src/streaming/Streaming.todo
@@ -0,0 +1,13 @@
+Come back to:
+☐ 404 handling
+☐ Rendermode
+☐ catch and fix stack trace (Dev) / send 500 (prod)
+✔ onShellReady vs onAllReady @done(23-08-16 20:53)
+☐ rs restart
+☐ make sure fast refresh still works in dev when you modify entry server
+☐ render modes? (isJsPage)
+☐ Why do we need an index.html for vite build?
+☐ Timeout on prod
+☐ Validate sending in query params, hash, etc. in
+  ☐ pages with params
+  ☐ pages without params

From 6bea3c60538f3961ea01f17048a4fc3405ce1798 Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Thu, 17 Aug 2023 20:14:37 +0700
Subject: [PATCH 08/14] Reload entry server on every request in dev

---
 packages/vite/src/streaming/Streaming.todo    |  7 +++----
 .../streaming/createReactStreamingHandler.ts  | 19 ++++++++++++-------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/packages/vite/src/streaming/Streaming.todo b/packages/vite/src/streaming/Streaming.todo
index 89421ea54f82..555d0f25d75f 100644
--- a/packages/vite/src/streaming/Streaming.todo
+++ b/packages/vite/src/streaming/Streaming.todo
@@ -1,11 +1,10 @@
 Come back to:
+✔ onShellReady vs onAllReady @done(23-08-16 20:53)
+✔ rs restart @done(23-08-17 20:06)
+✔ make sure refresh still works in dev when you modify entry server @done(23-08-17 20:13)
 ☐ 404 handling
 ☐ Rendermode
 ☐ catch and fix stack trace (Dev) / send 500 (prod)
-✔ onShellReady vs onAllReady @done(23-08-16 20:53)
-☐ rs restart
-☐ make sure fast refresh still works in dev when you modify entry server
-☐ render modes? (isJsPage)
 ☐ Why do we need an index.html for vite build?
 ☐ Timeout on prod
 ☐ Validate sending in query params, hash, etc. in
diff --git a/packages/vite/src/streaming/createReactStreamingHandler.ts b/packages/vite/src/streaming/createReactStreamingHandler.ts
index 2e134ef114ce..c259516ff78e 100644
--- a/packages/vite/src/streaming/createReactStreamingHandler.ts
+++ b/packages/vite/src/streaming/createReactStreamingHandler.ts
@@ -33,21 +33,26 @@ export const createReactStreamingHandler = async (
 
   let entryServerImport: any
 
-  if (viteDevServer) {
-    entryServerImport = await viteDevServer.ssrLoadModule(
-      rwPaths.web.entryServer as string // already validated in dev server
-    )
-  } else {
+  if (isProd) {
     entryServerImport = await import(rwPaths.web.distEntryServer)
   }
 
-  const ServerEntry = entryServerImport.ServerEntry || entryServerImport.default
-
   return async (req: Request, res: Response) => {
     if (redirect) {
       res.redirect(redirect.to)
     }
 
+    // Do this inside the handler for **dev-only**.
+    // This makes sure that changes to entry-server are picked up on refresh
+    if (!isProd) {
+      entryServerImport = await viteDevServer.ssrLoadModule(
+        rwPaths.web.entryServer as string // already validated in dev server
+      )
+    }
+
+    const ServerEntry =
+      entryServerImport.ServerEntry || entryServerImport.default
+
     const currentPathName = req.path
 
     let metaTags: TagDescriptor[] = []

From 63e31517aab469fd3519043665b640e7b2db8100 Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Thu, 17 Aug 2023 20:06:39 +0700
Subject: [PATCH 09/14] Allow restart on rs of dev web server

---
 packages/vite/src/devFeServer.ts | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/packages/vite/src/devFeServer.ts b/packages/vite/src/devFeServer.ts
index fa12d0c62345..2c22acfb564d 100644
--- a/packages/vite/src/devFeServer.ts
+++ b/packages/vite/src/devFeServer.ts
@@ -89,12 +89,14 @@ async function createServer() {
   return await app.listen(port)
 }
 
-createServer()
-
-// process.stdin.on('data', async (data) => {
-//   const str = data.toString().trim().toLowerCase()
-//   if (str === 'rs' || str === 'restart') {
-//     ;(await devApp).close()
-//     devApp = createServer()
-//   }
-// })
+let devApp = createServer()
+
+process.stdin.on('data', async (data) => {
+  const str = data.toString().trim().toLowerCase()
+  if (str === 'rs' || str === 'restart') {
+    console.log('Restarting dev web server.....')
+    ;(await devApp).close(() => {
+      devApp = createServer()
+    })
+  }
+})

From 94bce65767e5b1115faf7a6c00d987632a7face1 Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Thu, 17 Aug 2023 20:17:20 +0700
Subject: [PATCH 10/14] Remove old comments

---
 packages/internal/src/routes.ts | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/packages/internal/src/routes.ts b/packages/internal/src/routes.ts
index eb202df28424..a385018fce88 100644
--- a/packages/internal/src/routes.ts
+++ b/packages/internal/src/routes.ts
@@ -68,14 +68,14 @@ export function warningForDuplicateRoutes() {
 }
 
 export interface RWRouteManifestItem {
-  name: string // <-- AnalyzedRoute.name, RouteSpec.name
-  pathDefinition: string // <-- AnalyzedRoute.path, RouteSpec.path
-  matchRegexString: string | null // xAR, RouteSpec.matchRegexString
-  routeHooks: string | null // xAR, RouteSpec.routeHooks BUT in RouteSpec its the src path, here its the dist path
-  bundle: string | null // xAR, xRS
-  hasParams: boolean // xAR, RouteSpec.hasParams
-  redirect: { to: string; permanent: boolean } | null // xAR (not same type), RouteSpec.redirect
-  renderMode: 'html' | 'stream' // x, RouteSpec.renderMode
+  name: string
+  pathDefinition: string
+  matchRegexString: string | null
+  routeHooks: string | null
+  bundle: string | null
+  hasParams: boolean
+  redirect: { to: string; permanent: boolean } | null
+  renderMode: 'html' | 'stream'
   // Probably want isNotFound here, so we can attach a separate 404 handler
 }
 

From 580b27d523f97713004d28ea0696d16e4473f2ef Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Fri, 18 Aug 2023 19:41:29 +0700
Subject: [PATCH 11/14] Fix link construction in suspense client

---
 packages/web/src/apollo/links.tsx    | 117 +++++++++++++++++++
 packages/web/src/apollo/suspense.tsx | 162 ++++++++-------------------
 2 files changed, 163 insertions(+), 116 deletions(-)
 create mode 100644 packages/web/src/apollo/links.tsx

diff --git a/packages/web/src/apollo/links.tsx b/packages/web/src/apollo/links.tsx
new file mode 100644
index 000000000000..de9081c04b74
--- /dev/null
+++ b/packages/web/src/apollo/links.tsx
@@ -0,0 +1,117 @@
+import type { HttpOptions } from '@apollo/client'
+import { ApolloLink, HttpLink } from '@apollo/client'
+import { setContext } from '@apollo/client/link/context'
+import { print } from 'graphql/language/printer'
+
+export function createHttpLink(
+  uri: string,
+  httpLinkConfig: HttpOptions | undefined
+) {
+  return new HttpLink({
+    // @MARK: we have to construct the absoltue url for SSR
+    uri,
+    ...httpLinkConfig,
+    // you can disable result caching here if you want to
+    // (this does not work if you are rendering your page with `export const dynamic = "force-static"`)
+    fetchOptions: { cache: 'no-store' },
+  })
+}
+export function createUpdateDataLink(data: any) {
+  return new ApolloLink((operation, forward) => {
+    const { operationName, query, variables } = operation
+
+    data.mostRecentRequest = {}
+    data.mostRecentRequest.operationName = operationName
+    data.mostRecentRequest.operationKind = query?.kind.toString()
+    data.mostRecentRequest.variables = variables
+    data.mostRecentRequest.query = query && print(operation.query)
+
+    return forward(operation).map((result) => {
+      data.mostRecentResponse = result
+
+      return result
+    })
+  })
+}
+export function createAuthApolloLink(
+  authProviderType: string,
+  headers:
+    | {
+        'auth-provider'?: string | undefined
+        authorization?: string | undefined
+      }
+    | undefined
+) {
+  return new ApolloLink((operation, forward) => {
+    const { token } = operation.getContext()
+
+    // Only add auth headers when there's a token. `token` is `null` when `!isAuthenticated`.
+    const authHeaders = token
+      ? {
+          'auth-provider': authProviderType,
+          authorization: `Bearer ${token}`,
+        }
+      : {}
+
+    operation.setContext(() => ({
+      headers: {
+        ...operation.getContext().headers,
+        ...headers,
+        // Duped auth headers, because we may remove the `FetchConfigProvider` at a later date.
+        ...authHeaders,
+      },
+    }))
+
+    return forward(operation)
+  })
+}
+export function createTokenLink(getToken: () => Promise<string | null>) {
+  return setContext(async () => {
+    const token = await getToken()
+
+    return { token }
+  })
+}
+
+export function createFinalLink({
+  userPassedLink,
+  defaultLinks,
+}: {
+  userPassedLink?: ApolloLink | RedwoodApolloLinkFactory
+  defaultLinks: RedwoodApolloLinks
+}): ApolloLink {
+  if (userPassedLink) {
+    if (typeof userPassedLink === 'function') {
+      return userPassedLink(defaultLinks)
+    } else {
+      return userPassedLink
+    }
+  }
+
+  return ApolloLink.from(defaultLinks.map((l) => l.link))
+}
+
+// ~~~ Types ~~~
+
+export type RedwoodApolloLinkName =
+  | 'withToken'
+  | 'authMiddleware'
+  | 'updateDataApolloLink'
+  | 'httpLink'
+
+export type RedwoodApolloLink<
+  Name extends RedwoodApolloLinkName,
+  Link extends ApolloLink = ApolloLink
+> = {
+  name: Name
+  link: Link
+}
+
+export type RedwoodApolloLinks = [
+  RedwoodApolloLink<'withToken'>,
+  RedwoodApolloLink<'authMiddleware'>,
+  RedwoodApolloLink<'updateDataApolloLink'>,
+  RedwoodApolloLink<'httpLink', HttpLink>
+]
+
+export type RedwoodApolloLinkFactory = (links: RedwoodApolloLinks) => ApolloLink
diff --git a/packages/web/src/apollo/suspense.tsx b/packages/web/src/apollo/suspense.tsx
index 5ad29d3e8cbf..85a53198604d 100644
--- a/packages/web/src/apollo/suspense.tsx
+++ b/packages/web/src/apollo/suspense.tsx
@@ -9,26 +9,22 @@
 import type {
   ApolloCache,
   ApolloClientOptions,
+  HttpOptions,
+  InMemoryCacheConfig,
   setLogVerbosity,
 } from '@apollo/client'
-import * as apolloClient from '@apollo/client'
-import { setContext } from '@apollo/client/link/context'
+import {
+  ApolloLink,
+  setLogVerbosity as apolloSetLogVerbosity,
+  useMutation,
+  useSubscription,
+} from '@apollo/client'
 import {
   ApolloNextAppProvider,
   NextSSRApolloClient,
   NextSSRInMemoryCache,
   useSuspenseQuery,
 } from '@apollo/experimental-nextjs-app-support/ssr'
-import { print } from 'graphql/language/printer'
-
-// Note: Importing directly from `apollo/client` doesn't work properly in Storybook.
-const {
-  ApolloLink,
-  HttpLink,
-  useSubscription,
-  useMutation,
-  setLogVerbosity: apolloSetLogVerbosity,
-} = apolloClient
 
 import { UseAuth, useNoAuth } from '@redwoodjs/auth'
 import './typeOverride'
@@ -39,33 +35,29 @@ import {
 } from '../components/FetchConfigProvider'
 import { GraphQLHooksProvider } from '../components/GraphQLHooksProvider'
 
-export type ApolloClientCacheConfig = apolloClient.InMemoryCacheConfig
-
-export type RedwoodApolloLinkName =
-  | 'withToken'
-  | 'authMiddleware'
-  | 'updateDataApolloLink'
-  | 'httpLink'
-
-export type RedwoodApolloLink<
-  Name extends RedwoodApolloLinkName,
-  Link extends apolloClient.ApolloLink = apolloClient.ApolloLink
-> = {
-  name: Name
-  link: Link
+import type {
+  RedwoodApolloLink,
+  RedwoodApolloLinkFactory,
+  RedwoodApolloLinkName,
+  RedwoodApolloLinks,
+} from './links'
+import {
+  createAuthApolloLink,
+  createFinalLink,
+  createHttpLink,
+  createTokenLink,
+  createUpdateDataLink,
+} from './links'
+
+export type ApolloClientCacheConfig = InMemoryCacheConfig
+
+export type {
+  RedwoodApolloLink,
+  RedwoodApolloLinkFactory,
+  RedwoodApolloLinkName,
+  RedwoodApolloLinks,
 }
 
-export type RedwoodApolloLinks = [
-  RedwoodApolloLink<'withToken'>,
-  RedwoodApolloLink<'authMiddleware'>,
-  RedwoodApolloLink<'updateDataApolloLink'>,
-  RedwoodApolloLink<'httpLink', apolloClient.HttpLink>
-]
-
-export type RedwoodApolloLinkFactory = (
-  links: RedwoodApolloLinks
-) => apolloClient.ApolloLink
-
 export type GraphQLClientConfigProp = Omit<
   ApolloClientOptions<unknown>,
   'cache' | 'link'
@@ -88,7 +80,7 @@ export type GraphQLClientConfigProp = Omit<
    * }}>
    * ```
    */
-  httpLinkConfig?: apolloClient.HttpOptions
+  httpLinkConfig?: HttpOptions
   /**
    * Extend or overwrite `RedwoodApolloProvider`'s Apollo Link.
    *
@@ -112,7 +104,7 @@ export type GraphQLClientConfigProp = Omit<
    * - your function should return a single link (e.g., using `ApolloLink.from`; see https://www.apollographql.com/docs/react/api/link/introduction/#additive-composition)
    * - the `HttpLink` should come last (https://www.apollographql.com/docs/react/api/link/introduction/#the-terminating-link)
    */
-  link?: apolloClient.ApolloLink | RedwoodApolloLinkFactory
+  link?: ApolloLink | RedwoodApolloLinkFactory
 }
 
 const ApolloProviderWithFetchConfig: React.FunctionComponent<{
@@ -127,10 +119,6 @@ const ApolloProviderWithFetchConfig: React.FunctionComponent<{
   // See https://github.com/redwoodjs/redwood/issues/2473.
   apolloSetLogVerbosity(logLevel)
 
-  // Here we're using Apollo Link to customize Apollo Client's data flow.
-  // Although we're sending conventional HTTP-based requests and could just pass `uri` instead of `link`,
-  // we need to fetch a new token on every request, making middleware a good fit for this.
-  //
   // See https://www.apollographql.com/docs/react/api/link/introduction.
   const { getToken, type: authProviderType } = useAuth()
 
@@ -141,28 +129,6 @@ const ApolloProviderWithFetchConfig: React.FunctionComponent<{
     mostRecentResponse: undefined,
   } as any
 
-  const updateDataApolloLink = new ApolloLink((operation, forward) => {
-    const { operationName, query, variables } = operation
-
-    data.mostRecentRequest = {}
-    data.mostRecentRequest.operationName = operationName
-    data.mostRecentRequest.operationKind = query?.kind.toString()
-    data.mostRecentRequest.variables = variables
-    data.mostRecentRequest.query = query && print(operation.query)
-
-    return forward(operation).map((result) => {
-      data.mostRecentResponse = result
-
-      return result
-    })
-  })
-
-  const withToken = setContext(async () => {
-    const token = await getToken()
-
-    return { token }
-  })
-
   const { headers, uri } = useFetchConfig()
 
   const getGraphqlUrl = () => {
@@ -177,51 +143,20 @@ const ApolloProviderWithFetchConfig: React.FunctionComponent<{
       : uri
   }
 
-  const authMiddleware = new ApolloLink((operation, forward) => {
-    const { token } = operation.getContext()
-
-    // Only add auth headers when there's a token. `token` is `null` when `!isAuthenticated`.
-    const authHeaders = token
-      ? {
-          'auth-provider': authProviderType,
-          authorization: `Bearer ${token}`,
-        }
-      : {}
-
-    operation.setContext(() => ({
-      headers: {
-        ...operation.getContext().headers,
-        ...headers,
-        // Duped auth headers, because we may remove the `FetchConfigProvider` at a later date.
-        ...authHeaders,
-      },
-    }))
-
-    return forward(operation)
-  })
+  const { httpLinkConfig, link: userPassedLink, ...otherConfig } = config ?? {}
 
-  const { httpLinkConfig, link: redwoodApolloLink, ...rest } = config ?? {}
-
-  // A terminating link. Apollo Client uses this to send GraphQL operations to a server over HTTP.
-  // See https://www.apollographql.com/docs/react/api/link/introduction/#the-terminating-link.
-  const httpLink = new HttpLink({ uri, ...httpLinkConfig })
-
-  // The order here is important. The last link *must* be a terminating link like HttpLink.
+  // We use this object, because that's the shape of what we pass to the config.link factory
   const redwoodApolloLinks: RedwoodApolloLinks = [
-    { name: 'withToken', link: withToken },
-    { name: 'authMiddleware', link: authMiddleware },
-    { name: 'updateDataApolloLink', link: updateDataApolloLink },
-    { name: 'httpLink', link: httpLink },
+    { name: 'withToken', link: createTokenLink(getToken) },
+    {
+      name: 'authMiddleware',
+      link: createAuthApolloLink(authProviderType, headers),
+    },
+    // @TODO: do we need this in prod? I think it's only for dev errors
+    { name: 'updateDataApolloLink', link: createUpdateDataLink(data) },
+    { name: 'httpLink', link: createHttpLink(getGraphqlUrl(), httpLinkConfig) },
   ]
 
-  let link = redwoodApolloLink
-
-  link ??= ApolloLink.from(redwoodApolloLinks.map((l) => l.link))
-
-  if (typeof link === 'function') {
-    link = link(redwoodApolloLinks)
-  }
-
   const extendErrorAndRethrow = (error: any, _errorInfo: React.ErrorInfo) => {
     error['mostRecentRequest'] = data.mostRecentRequest
     error['mostRecentResponse'] = data.mostRecentResponse
@@ -229,18 +164,13 @@ const ApolloProviderWithFetchConfig: React.FunctionComponent<{
   }
 
   function makeClient() {
-    const httpLink = new HttpLink({
-      // @MARK: we have to construct the absoltue url for SSR
-      uri: getGraphqlUrl(),
-      // you can disable result caching here if you want to
-      // (this does not work if you are rendering your page with `export const dynamic = "force-static"`)
-      fetchOptions: { cache: 'no-store' },
-    })
-
     // @MARK use special Apollo client
     return new NextSSRApolloClient({
-      link: httpLink,
-      ...rest,
+      link: createFinalLink({
+        userPassedLink,
+        defaultLinks: redwoodApolloLinks,
+      }),
+      ...otherConfig,
     })
   }
 

From c57c5b78ce39ec3ff972b9e0216dab8e99cf32a8 Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Fri, 18 Aug 2023 20:00:30 +0700
Subject: [PATCH 12/14] Add option to skip adding fw dependencies in
 proejct:sync

---
 tasks/framework-tools/frameworkSyncToProject.mjs | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/tasks/framework-tools/frameworkSyncToProject.mjs b/tasks/framework-tools/frameworkSyncToProject.mjs
index fb8af9ac88d2..9b21ee65bb8e 100644
--- a/tasks/framework-tools/frameworkSyncToProject.mjs
+++ b/tasks/framework-tools/frameworkSyncToProject.mjs
@@ -61,6 +61,12 @@ async function main() {
       type: 'boolean',
       default: true,
     })
+    .option('addFwDeps', {
+      description:
+        'Modify the projects package.json to include fw dependencies',
+      type: 'boolean',
+      default: true,
+    })
     .option('watch', {
       description: 'Watch for changes to the framework packages',
       type: 'boolean',
@@ -162,8 +168,13 @@ async function main() {
       process.on('exit', cleanUp)
     }
 
-    logStatus("Adding the Redwood framework's dependencies...")
-    addDependenciesToPackageJson(redwoodProjectPackageJsonPath)
+    if (options.addFwDeps) {
+      // Rare case, but sometimes we don't want to modify any dependency versions
+      logStatus("Adding the Redwood framework's dependencies...")
+      addDependenciesToPackageJson(redwoodProjectPackageJsonPath)
+    } else {
+      logStatus("Skipping adding framework's dependencies...")
+    }
 
     try {
       execSync('yarn install', {

From 8441daa79ed5b0ebfef05371690a8a23957269d3 Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Fri, 18 Aug 2023 20:08:55 +0700
Subject: [PATCH 13/14] Remove todo file

---
 packages/vite/src/streaming/Streaming.todo | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 packages/vite/src/streaming/Streaming.todo

diff --git a/packages/vite/src/streaming/Streaming.todo b/packages/vite/src/streaming/Streaming.todo
deleted file mode 100644
index 555d0f25d75f..000000000000
--- a/packages/vite/src/streaming/Streaming.todo
+++ /dev/null
@@ -1,12 +0,0 @@
-Come back to:
-✔ onShellReady vs onAllReady @done(23-08-16 20:53)
-✔ rs restart @done(23-08-17 20:06)
-✔ make sure refresh still works in dev when you modify entry server @done(23-08-17 20:13)
-☐ 404 handling
-☐ Rendermode
-☐ catch and fix stack trace (Dev) / send 500 (prod)
-☐ Why do we need an index.html for vite build?
-☐ Timeout on prod
-☐ Validate sending in query params, hash, etc. in
-  ☐ pages with params
-  ☐ pages without params

From f23c502f2e2532e57f02705e2f14247908993458 Mon Sep 17 00:00:00 2001
From: Daniel Choudhury <dannychoudhury@gmail.com>
Date: Fri, 18 Aug 2023 20:17:22 +0700
Subject: [PATCH 14/14] Rename createFinalLink argument

---
 packages/web/src/apollo/links.tsx    | 12 ++++++------
 packages/web/src/apollo/suspense.tsx |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/web/src/apollo/links.tsx b/packages/web/src/apollo/links.tsx
index de9081c04b74..04721652d896 100644
--- a/packages/web/src/apollo/links.tsx
+++ b/packages/web/src/apollo/links.tsx
@@ -74,17 +74,17 @@ export function createTokenLink(getToken: () => Promise<string | null>) {
 }
 
 export function createFinalLink({
-  userPassedLink,
+  userConfiguredLink,
   defaultLinks,
 }: {
-  userPassedLink?: ApolloLink | RedwoodApolloLinkFactory
+  userConfiguredLink?: ApolloLink | RedwoodApolloLinkFactory
   defaultLinks: RedwoodApolloLinks
 }): ApolloLink {
-  if (userPassedLink) {
-    if (typeof userPassedLink === 'function') {
-      return userPassedLink(defaultLinks)
+  if (userConfiguredLink) {
+    if (typeof userConfiguredLink === 'function') {
+      return userConfiguredLink(defaultLinks)
     } else {
-      return userPassedLink
+      return userConfiguredLink
     }
   }
 
diff --git a/packages/web/src/apollo/suspense.tsx b/packages/web/src/apollo/suspense.tsx
index 85a53198604d..838ad998ac53 100644
--- a/packages/web/src/apollo/suspense.tsx
+++ b/packages/web/src/apollo/suspense.tsx
@@ -167,7 +167,7 @@ const ApolloProviderWithFetchConfig: React.FunctionComponent<{
     // @MARK use special Apollo client
     return new NextSSRApolloClient({
       link: createFinalLink({
-        userPassedLink,
+        userConfiguredLink: userPassedLink,
         defaultLinks: redwoodApolloLinks,
       }),
       ...otherConfig,