fix(gatsby-adapter-netlify): handle cases with large cached _redirects and/or _headers files (#38559)

packages/gatsby-adapter-netlify/src/__tests__/route-handler.ts

@@ -0,0 +1,145 @@
+import fs from "fs-extra"
+import { tmpdir } from "os"
+import { join } from "path"
+import {
+  injectEntries,
+  ADAPTER_MARKER_START,
+  ADAPTER_MARKER_END,
+  NETLIFY_PLUGIN_MARKER_START,
+  NETLIFY_PLUGIN_MARKER_END,
+  GATSBY_PLUGIN_MARKER_START,
+} from "../route-handler"
+
+function generateLotOfContent(placeholderCharacter: string): string {
+  return (placeholderCharacter.repeat(80) + `\n`).repeat(1_000_000)
+}
+
+const newAdapterContent = generateLotOfContent(`a`)
+const previousAdapterContent =
+  ADAPTER_MARKER_START +
+  `\n` +
+  generateLotOfContent(`b`) +
+  ADAPTER_MARKER_END +
+  `\n`
+
+const gatsbyPluginNetlifyContent =
+  GATSBY_PLUGIN_MARKER_START + `\n` + generateLotOfContent(`c`)
+
+const netlifyPluginGatsbyContent =
+  NETLIFY_PLUGIN_MARKER_START +
+  `\n` +
+  generateLotOfContent(`c`) +
+  NETLIFY_PLUGIN_MARKER_END +
+  `\n`
+
+const customContent1 =
+  `# customContent1 start` +
+  `\n` +
+  generateLotOfContent(`x`) +
+  `# customContent1 end` +
+  `\n`
+const customContent2 =
+  `# customContent2 start` +
+  `\n` +
+  generateLotOfContent(`y`) +
+  `# customContent2 end` +
+  `\n`
+const customContent3 =
+  `# customContent3 start` +
+  `\n` +
+  generateLotOfContent(`z`) +
+  `# customContent3 end` +
+  `\n`
+
+async function getContent(previousContent?: string): Promise<string> {
+  const filePath = join(
+    await fs.mkdtemp(join(tmpdir(), `inject-entries`)),
+    `out.txt`
+  )
+
+  if (typeof previousContent !== `undefined`) {
+    await fs.writeFile(filePath, previousContent)
+  }
+
+  await injectEntries(filePath, newAdapterContent)
+
+  return fs.readFile(filePath, `utf8`)
+}
+
+jest.setTimeout(60_000)
+
+describe(`route-handler`, () => {
+  describe(`injectEntries`, () => {
+    it(`no cached file`, async () => {
+      const content = await getContent()
+
+      expect(content.indexOf(newAdapterContent)).not.toBe(-1)
+    })
+
+    describe(`has cached file`, () => {
+      it(`no previous adapter or plugins or custom entries`, async () => {
+        const content = await getContent(``)
+
+        expect(content.indexOf(newAdapterContent)).not.toBe(-1)
+      })
+
+      it(`has just custom entries`, async () => {
+        const content = await getContent(customContent1)
+
+        expect(content.indexOf(newAdapterContent)).not.toBe(-1)
+        expect(content.indexOf(customContent1)).not.toBe(-1)
+      })
+
+      it(`has just gatsby-plugin-netlify entries`, async () => {
+        const content = await getContent(gatsbyPluginNetlifyContent)
+
+        expect(content.indexOf(newAdapterContent)).not.toBe(-1)
+        // it removes gatsby-plugin-netlify entries
+        expect(content.indexOf(GATSBY_PLUGIN_MARKER_START)).toBe(-1)
+        expect(content.indexOf(gatsbyPluginNetlifyContent)).toBe(-1)
+      })
+
+      it(`has just netlify-plugin-gatsby entries`, async () => {
+        const content = await getContent(netlifyPluginGatsbyContent)
+
+        expect(content.indexOf(newAdapterContent)).not.toBe(-1)
+        // it removes netlify-plugin-gatsby entries
+        expect(content.indexOf(NETLIFY_PLUGIN_MARKER_START)).toBe(-1)
+        expect(content.indexOf(NETLIFY_PLUGIN_MARKER_END)).toBe(-1)
+        expect(content.indexOf(netlifyPluginGatsbyContent)).toBe(-1)
+      })
+
+      it(`has gatsby-plugin-netlify, nelify-plugin-gatsby, custom content and previous adapter content`, async () => {
+        // kitchen-sink
+        const previousContent =
+          customContent1 +
+          previousAdapterContent +
+          customContent2 +
+          netlifyPluginGatsbyContent +
+          customContent3 +
+          gatsbyPluginNetlifyContent
+
+        const content = await getContent(previousContent)
+
+        expect(content.indexOf(newAdapterContent)).not.toBe(-1)
+
+        // it preserve any custom entries
+        expect(content.indexOf(customContent1)).not.toBe(-1)
+        expect(content.indexOf(customContent2)).not.toBe(-1)
+        expect(content.indexOf(customContent3)).not.toBe(-1)
+
+        // it removes previous gatsby-adapter-netlify entries
+        expect(content.indexOf(previousAdapterContent)).toBe(-1)
+
+        // it removes gatsby-plugin-netlify entries
+        expect(content.indexOf(GATSBY_PLUGIN_MARKER_START)).toBe(-1)
+        expect(content.indexOf(gatsbyPluginNetlifyContent)).toBe(-1)
+
+        // it removes netlify-plugin-gatsby entries
+        expect(content.indexOf(NETLIFY_PLUGIN_MARKER_START)).toBe(-1)
+        expect(content.indexOf(NETLIFY_PLUGIN_MARKER_END)).toBe(-1)
+        expect(content.indexOf(netlifyPluginGatsbyContent)).toBe(-1)
+      })
+    })
+  })
+})

packages/gatsby-adapter-netlify/src/route-handler.ts

@@ -1,5 +1,7 @@
 import type { RoutesManifest } from "gatsby"
-import { EOL } from "os"
+import { tmpdir } from "os"
+import { Transform } from "stream"
+import { join, basename } from "path"
 import fs from "fs-extra"
 
 const NETLIFY_REDIRECT_KEYWORDS_ALLOWLIST = new Set([
@@ -20,35 +22,112 @@ const toNetlifyPath = (fromPath: string, toPath: string): Array<string> => {
 
   return [netlifyFromPath, netlifyToPath]
 }
-const MARKER_START = `# gatsby-adapter-netlify start`
-const MARKER_END = `# gatsby-adapter-netlify end`
-
-async function injectEntries(fileName: string, content: string): Promise<void> {
+export const ADAPTER_MARKER_START = `# gatsby-adapter-netlify start`
+export const ADAPTER_MARKER_END = `# gatsby-adapter-netlify end`
+export const NETLIFY_PLUGIN_MARKER_START = `# @netlify/plugin-gatsby redirects start`
+export const NETLIFY_PLUGIN_MARKER_END = `# @netlify/plugin-gatsby redirects end`
+export const GATSBY_PLUGIN_MARKER_START = `## Created with gatsby-plugin-netlify`
+
+export async function injectEntries(
+  fileName: string,
+  content: string
+): Promise<void> {
   await fs.ensureFile(fileName)
 
-  const data = await fs.readFile(fileName, `utf8`)
-  const [initial = ``, rest = ``] = data.split(MARKER_START)
-  const [, final = ``] = rest.split(MARKER_END)
-  const out = [
-    initial === EOL ? `` : initial,
-    initial.endsWith(EOL) ? `` : EOL,
-    MARKER_START,
-    EOL,
-    content,
-    EOL,
-    MARKER_END,
-    final.startsWith(EOL) ? `` : EOL,
-    final === EOL ? `` : final,
-  ]
-    .filter(Boolean)
-    .join(``)
-    .replace(
-      /# @netlify\/plugin-gatsby redirects start(.|\n|\r)*# @netlify\/plugin-gatsby redirects end/gm,
-      ``
-    )
-    .replace(/## Created with gatsby-plugin-netlify(.|\n|\r)*$/gm, ``)
-
-  await fs.outputFile(fileName, out)
+  const tmpFile = join(
+    await fs.mkdtemp(join(tmpdir(), basename(fileName))),
+    `out.txt`
+  )
+
+  let tail = ``
+  let insideNetlifyPluginGatsby = false
+  let insideGatsbyPluginNetlify = false
+  let insideGatsbyAdapterNetlify = false
+  let injectedEntries = false
+
+  const annotatedContent = `${ADAPTER_MARKER_START}\n${content}\n${ADAPTER_MARKER_END}\n`
+
+  function getContentToAdd(final: boolean): string {
+    const lines = tail.split(`\n`)
+    tail = ``
+
+    let contentToAdd = ``
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i]
+
+      if (!final && i === lines.length - 1) {
+        tail = line
+        break
+      }
+
+      let skipLine =
+        insideGatsbyAdapterNetlify ||
+        insideGatsbyPluginNetlify ||
+        insideNetlifyPluginGatsby
+
+      if (line.includes(ADAPTER_MARKER_START)) {
+        skipLine = true
+        insideGatsbyAdapterNetlify = true
+      } else if (line.includes(ADAPTER_MARKER_END)) {
+        insideGatsbyAdapterNetlify = false
+        contentToAdd += annotatedContent
+        injectedEntries = true
+      } else if (line.includes(NETLIFY_PLUGIN_MARKER_START)) {
+        insideNetlifyPluginGatsby = true
+        skipLine = true
+      } else if (line.includes(NETLIFY_PLUGIN_MARKER_END)) {
+        insideNetlifyPluginGatsby = false
+      } else if (line.includes(GATSBY_PLUGIN_MARKER_START)) {
+        insideGatsbyPluginNetlify = true
+        skipLine = true
+      }
+
+      if (!skipLine) {
+        contentToAdd += line + `\n`
+      }
+    }
+
+    return contentToAdd
+  }
+
+  const streamReplacer = new Transform({
+    transform(chunk, _encoding, callback): void {
+      tail = tail + chunk.toString()
+
+      try {
+        callback(null, getContentToAdd(false))
+      } catch (e) {
+        callback(e)
+      }
+    },
+    flush(callback): void {
+      try {
+        let contentToAdd = getContentToAdd(true)
+        if (!injectedEntries) {
+          contentToAdd += annotatedContent
+        }
+        callback(null, contentToAdd)
+      } catch (e) {
+        callback(e)
+      }
+    },
+  })
+
+  await new Promise<void>((resolve, reject) => {
+    const writeStream = fs.createWriteStream(tmpFile)
+    const pipeline = fs
+      .createReadStream(fileName)
+      .pipe(streamReplacer)
+      .pipe(writeStream)
+
+    pipeline.on(`finish`, resolve)
+    pipeline.on(`error`, reject)
+    streamReplacer.on(`error`, reject)
+  })
+
+  // remove previous file and move new file from tmp to final path
+  await fs.remove(fileName)
+  await fs.move(tmpFile, fileName)
 }
 
 export async function handleRoutesManifest(