From 34d6f9fcdfc4ad8be0b74f13444a00ff5917cf73 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 19 Sep 2023 15:37:09 +0200 Subject: [PATCH] fix(gatsby-adapter-netlify): handle cases with large cached _redirects and/or _headers files (#38559) * test(gatsby-adapter-netlify): add unit tests for entries injection * fix(gatsby-adapter-netlify): handle cases with large cached _redirects and/or _headers filestest(gatsby-adapter-netlify): add unit tests for entries injection * Update route-handler.ts (cherry picked from commit db41d1356c527cf4028142050978accd4abb1e9a) --- .../src/__tests__/route-handler.ts | 145 ++++++++++++++++++ .../src/route-handler.ts | 135 ++++++++++++---- 2 files changed, 252 insertions(+), 28 deletions(-) create mode 100644 packages/gatsby-adapter-netlify/src/__tests__/route-handler.ts diff --git a/packages/gatsby-adapter-netlify/src/__tests__/route-handler.ts b/packages/gatsby-adapter-netlify/src/__tests__/route-handler.ts new file mode 100644 index 0000000000000..9d8af395b6c97 --- /dev/null +++ b/packages/gatsby-adapter-netlify/src/__tests__/route-handler.ts @@ -0,0 +1,145 @@ +import fs from "fs-extra" +import { tmpdir } from "os" +import { join } from "path" +import { + injectEntries, + ADAPTER_MARKER_START, + ADAPTER_MARKER_END, + NETLIFY_PLUGIN_MARKER_START, + NETLIFY_PLUGIN_MARKER_END, + GATSBY_PLUGIN_MARKER_START, +} from "../route-handler" + +function generateLotOfContent(placeholderCharacter: string): string { + return (placeholderCharacter.repeat(80) + `\n`).repeat(1_000_000) +} + +const newAdapterContent = generateLotOfContent(`a`) +const previousAdapterContent = + ADAPTER_MARKER_START + + `\n` + + generateLotOfContent(`b`) + + ADAPTER_MARKER_END + + `\n` + +const gatsbyPluginNetlifyContent = + GATSBY_PLUGIN_MARKER_START + `\n` + generateLotOfContent(`c`) + +const netlifyPluginGatsbyContent = + NETLIFY_PLUGIN_MARKER_START + + `\n` + + generateLotOfContent(`c`) + + NETLIFY_PLUGIN_MARKER_END + + `\n` + +const customContent1 = + `# customContent1 start` + + `\n` + + generateLotOfContent(`x`) + + `# customContent1 end` + + `\n` +const customContent2 = + `# customContent2 start` + + `\n` + + generateLotOfContent(`y`) + + `# customContent2 end` + + `\n` +const customContent3 = + `# customContent3 start` + + `\n` + + generateLotOfContent(`z`) + + `# customContent3 end` + + `\n` + +async function getContent(previousContent?: string): Promise { + const filePath = join( + await fs.mkdtemp(join(tmpdir(), `inject-entries`)), + `out.txt` + ) + + if (typeof previousContent !== `undefined`) { + await fs.writeFile(filePath, previousContent) + } + + await injectEntries(filePath, newAdapterContent) + + return fs.readFile(filePath, `utf8`) +} + +jest.setTimeout(60_000) + +describe(`route-handler`, () => { + describe(`injectEntries`, () => { + it(`no cached file`, async () => { + const content = await getContent() + + expect(content.indexOf(newAdapterContent)).not.toBe(-1) + }) + + describe(`has cached file`, () => { + it(`no previous adapter or plugins or custom entries`, async () => { + const content = await getContent(``) + + expect(content.indexOf(newAdapterContent)).not.toBe(-1) + }) + + it(`has just custom entries`, async () => { + const content = await getContent(customContent1) + + expect(content.indexOf(newAdapterContent)).not.toBe(-1) + expect(content.indexOf(customContent1)).not.toBe(-1) + }) + + it(`has just gatsby-plugin-netlify entries`, async () => { + const content = await getContent(gatsbyPluginNetlifyContent) + + expect(content.indexOf(newAdapterContent)).not.toBe(-1) + // it removes gatsby-plugin-netlify entries + expect(content.indexOf(GATSBY_PLUGIN_MARKER_START)).toBe(-1) + expect(content.indexOf(gatsbyPluginNetlifyContent)).toBe(-1) + }) + + it(`has just netlify-plugin-gatsby entries`, async () => { + const content = await getContent(netlifyPluginGatsbyContent) + + expect(content.indexOf(newAdapterContent)).not.toBe(-1) + // it removes netlify-plugin-gatsby entries + expect(content.indexOf(NETLIFY_PLUGIN_MARKER_START)).toBe(-1) + expect(content.indexOf(NETLIFY_PLUGIN_MARKER_END)).toBe(-1) + expect(content.indexOf(netlifyPluginGatsbyContent)).toBe(-1) + }) + + it(`has gatsby-plugin-netlify, nelify-plugin-gatsby, custom content and previous adapter content`, async () => { + // kitchen-sink + const previousContent = + customContent1 + + previousAdapterContent + + customContent2 + + netlifyPluginGatsbyContent + + customContent3 + + gatsbyPluginNetlifyContent + + const content = await getContent(previousContent) + + expect(content.indexOf(newAdapterContent)).not.toBe(-1) + + // it preserve any custom entries + expect(content.indexOf(customContent1)).not.toBe(-1) + expect(content.indexOf(customContent2)).not.toBe(-1) + expect(content.indexOf(customContent3)).not.toBe(-1) + + // it removes previous gatsby-adapter-netlify entries + expect(content.indexOf(previousAdapterContent)).toBe(-1) + + // it removes gatsby-plugin-netlify entries + expect(content.indexOf(GATSBY_PLUGIN_MARKER_START)).toBe(-1) + expect(content.indexOf(gatsbyPluginNetlifyContent)).toBe(-1) + + // it removes netlify-plugin-gatsby entries + expect(content.indexOf(NETLIFY_PLUGIN_MARKER_START)).toBe(-1) + expect(content.indexOf(NETLIFY_PLUGIN_MARKER_END)).toBe(-1) + expect(content.indexOf(netlifyPluginGatsbyContent)).toBe(-1) + }) + }) + }) +}) diff --git a/packages/gatsby-adapter-netlify/src/route-handler.ts b/packages/gatsby-adapter-netlify/src/route-handler.ts index 17c5b8f346aa4..d9e5a6b6d84a2 100644 --- a/packages/gatsby-adapter-netlify/src/route-handler.ts +++ b/packages/gatsby-adapter-netlify/src/route-handler.ts @@ -1,5 +1,7 @@ import type { RoutesManifest } from "gatsby" -import { EOL } from "os" +import { tmpdir } from "os" +import { Transform } from "stream" +import { join, basename } from "path" import fs from "fs-extra" const NETLIFY_REDIRECT_KEYWORDS_ALLOWLIST = new Set([ @@ -20,35 +22,112 @@ const toNetlifyPath = (fromPath: string, toPath: string): Array => { return [netlifyFromPath, netlifyToPath] } -const MARKER_START = `# gatsby-adapter-netlify start` -const MARKER_END = `# gatsby-adapter-netlify end` - -async function injectEntries(fileName: string, content: string): Promise { +export const ADAPTER_MARKER_START = `# gatsby-adapter-netlify start` +export const ADAPTER_MARKER_END = `# gatsby-adapter-netlify end` +export const NETLIFY_PLUGIN_MARKER_START = `# @netlify/plugin-gatsby redirects start` +export const NETLIFY_PLUGIN_MARKER_END = `# @netlify/plugin-gatsby redirects end` +export const GATSBY_PLUGIN_MARKER_START = `## Created with gatsby-plugin-netlify` + +export async function injectEntries( + fileName: string, + content: string +): Promise { await fs.ensureFile(fileName) - const data = await fs.readFile(fileName, `utf8`) - const [initial = ``, rest = ``] = data.split(MARKER_START) - const [, final = ``] = rest.split(MARKER_END) - const out = [ - initial === EOL ? `` : initial, - initial.endsWith(EOL) ? `` : EOL, - MARKER_START, - EOL, - content, - EOL, - MARKER_END, - final.startsWith(EOL) ? `` : EOL, - final === EOL ? `` : final, - ] - .filter(Boolean) - .join(``) - .replace( - /# @netlify\/plugin-gatsby redirects start(.|\n|\r)*# @netlify\/plugin-gatsby redirects end/gm, - `` - ) - .replace(/## Created with gatsby-plugin-netlify(.|\n|\r)*$/gm, ``) - - await fs.outputFile(fileName, out) + const tmpFile = join( + await fs.mkdtemp(join(tmpdir(), basename(fileName))), + `out.txt` + ) + + let tail = `` + let insideNetlifyPluginGatsby = false + let insideGatsbyPluginNetlify = false + let insideGatsbyAdapterNetlify = false + let injectedEntries = false + + const annotatedContent = `${ADAPTER_MARKER_START}\n${content}\n${ADAPTER_MARKER_END}\n` + + function getContentToAdd(final: boolean): string { + const lines = tail.split(`\n`) + tail = `` + + let contentToAdd = `` + for (let i = 0; i < lines.length; i++) { + const line = lines[i] + + if (!final && i === lines.length - 1) { + tail = line + break + } + + let skipLine = + insideGatsbyAdapterNetlify || + insideGatsbyPluginNetlify || + insideNetlifyPluginGatsby + + if (line.includes(ADAPTER_MARKER_START)) { + skipLine = true + insideGatsbyAdapterNetlify = true + } else if (line.includes(ADAPTER_MARKER_END)) { + insideGatsbyAdapterNetlify = false + contentToAdd += annotatedContent + injectedEntries = true + } else if (line.includes(NETLIFY_PLUGIN_MARKER_START)) { + insideNetlifyPluginGatsby = true + skipLine = true + } else if (line.includes(NETLIFY_PLUGIN_MARKER_END)) { + insideNetlifyPluginGatsby = false + } else if (line.includes(GATSBY_PLUGIN_MARKER_START)) { + insideGatsbyPluginNetlify = true + skipLine = true + } + + if (!skipLine) { + contentToAdd += line + `\n` + } + } + + return contentToAdd + } + + const streamReplacer = new Transform({ + transform(chunk, _encoding, callback): void { + tail = tail + chunk.toString() + + try { + callback(null, getContentToAdd(false)) + } catch (e) { + callback(e) + } + }, + flush(callback): void { + try { + let contentToAdd = getContentToAdd(true) + if (!injectedEntries) { + contentToAdd += annotatedContent + } + callback(null, contentToAdd) + } catch (e) { + callback(e) + } + }, + }) + + await new Promise((resolve, reject) => { + const writeStream = fs.createWriteStream(tmpFile) + const pipeline = fs + .createReadStream(fileName) + .pipe(streamReplacer) + .pipe(writeStream) + + pipeline.on(`finish`, resolve) + pipeline.on(`error`, reject) + streamReplacer.on(`error`, reject) + }) + + // remove previous file and move new file from tmp to final path + await fs.remove(fileName) + await fs.move(tmpFile, fileName) } export async function handleRoutesManifest(