SSG: partial builds (#689)

Introduces a `--partial` flag that will skip rendering of already
existing static RSC and HTML files.

## Use case

Content updates of larger static sites should not re-render every page.

## Notes

* Consciously avoids dealing with deleting files (cache invalidation). I
think this can be solved much better with userland domain knowledge.
* Bundle builds should probably be skipped as well, but are too tangled
at this point. At least they don't scale with the amount of static
pages.

---------

Co-authored-by: Daishi Kato <[email protected]>

e2e/fixtures/partial-build/README.md

@@ -0,0 +1 @@
+# Partial builds for static websites

e2e/fixtures/partial-build/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "partial-build",
+  "version": "0.1.0",
+  "type": "module",
+  "private": true,
+  "scripts": {
+    "dev": "waku dev",
+    "build": "waku build",
+    "start": "waku start",
+    "partial": "waku build --experimental-partial"
+  },
+  "dependencies": {
+    "react": "19.0.0-beta-4508873393-20240430",
+    "react-dom": "19.0.0-beta-4508873393-20240430",
+    "react-server-dom-webpack": "19.0.0-beta-4508873393-20240430",
+    "waku": "workspace:*",
+    "serve": "^14.2.3"
+  },
+  "devDependencies": {
+    "@types/react": "18.3.1",
+    "@types/react-dom": "18.3.0",
+    "typescript": "5.4.4"
+  }
+}

e2e/fixtures/partial-build/src/pages/index.tsx

@@ -0,0 +1,9 @@
+export default function HomePage() {
+  return <div>Home</div>;
+}
+
+export async function getConfig() {
+  return {
+    render: 'static',
+  };
+}

e2e/fixtures/partial-build/src/pages/page/[title].tsx

@@ -0,0 +1,12 @@
+import { getEnv } from 'waku/server';
+
+export default function Test({ title }: { title: string }) {
+  return <div data-testid="title">{title}</div>;
+}
+
+export async function getConfig() {
+  return {
+    render: 'static',
+    staticPaths: getEnv('PAGES')?.split(',') || [],
+  };
+}

e2e/fixtures/partial-build/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "strict": true,
+    "target": "esnext",
+    "downlevelIteration": true,
+    "esModuleInterop": true,
+    "module": "nodenext",
+    "skipLibCheck": true,
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": true,
+    "types": ["react/experimental"],
+    "jsx": "react-jsx",
+    "rootDir": "./src",
+    "outDir": "./dist"
+  }
+}

e2e/partial-builds.spec.ts

@@ -0,0 +1,76 @@
+import { execSync, exec, ChildProcess } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import waitPort from 'wait-port';
+import { getFreePort, terminate, test } from './utils.js';
+import { rm } from 'node:fs/promises';
+import { expect } from '@playwright/test';
+import { statSync } from 'fs';
+
+const cwd = fileURLToPath(new URL('./fixtures/partial-build', import.meta.url));
+
+const waku = fileURLToPath(
+  new URL('../packages/waku/dist/cli.js', import.meta.url),
+);
+
+test.describe(`partial builds`, () => {
+  test.skip(
+    ({ browserName }) => browserName !== 'chromium',
+    'Browsers are not relevant for this test. One is enough.',
+  );
+
+  let cp: ChildProcess;
+  let port: number;
+
+  test.beforeEach(async ({ page }) => {
+    await rm(`${cwd}/dist`, {
+      recursive: true,
+      force: true,
+    });
+    execSync(`node ${waku} build`, {
+      cwd,
+      env: { ...process.env, PAGES: 'a' },
+    });
+    port = await getFreePort();
+    cp = exec(`node ${waku} start --port ${port}`, { cwd });
+    await waitPort({ port });
+    await page.goto(`http://localhost:${port}/page/a`);
+    expect(await page.getByTestId('title').textContent()).toBe('a');
+  });
+
+  test('does not change pages that already exist', async () => {
+    const htmlBefore = statSync(`${cwd}/dist/public/page/a/index.html`);
+    const rscBefore = statSync(`${cwd}/dist/public/RSC/page/a.txt`);
+    execSync(`node ${waku} build --experimental-partial`, {
+      cwd,
+      env: { ...process.env, PAGES: 'a,b' },
+    });
+    const htmlAfter = statSync(`${cwd}/dist/public/page/a/index.html`);
+    const rscAfter = statSync(`${cwd}/dist/public/RSC/page/a.txt`);
+    expect(htmlBefore.mtimeMs).toBe(htmlAfter.mtimeMs);
+    expect(rscBefore.mtimeMs).toBe(rscAfter.mtimeMs);
+  });
+
+  test('adds new pages', async ({ page }) => {
+    execSync(`node ${waku} build --experimental-partial`, {
+      cwd,
+      env: { ...process.env, PAGES: 'a,b' },
+    });
+    await page.goto(`http://localhost:${port}/page/b`);
+    expect(await page.getByTestId('title').textContent()).toBe('b');
+  });
+
+  test('does not delete old pages', async ({ page }) => {
+    execSync(`node ${waku} build --experimental-partial`, {
+      cwd,
+      env: { ...process.env, PAGES: 'c' },
+    });
+    await page.goto(`http://localhost:${port}/page/a`);
+    expect(await page.getByTestId('title').textContent()).toBe('a');
+    await page.goto(`http://localhost:${port}/page/c`);
+    expect(await page.getByTestId('title').textContent()).toBe('c');
+  });
+
+  test.afterEach(async () => {
+    await terminate(cp.pid!);
+  });
+});

packages/waku/src/cli.ts

@@ -47,6 +47,9 @@ const { values, positionals } = parseArgs({
     'with-aws-lambda': {
       type: 'boolean',
     },
+    'experimental-partial': {
+      type: 'boolean',
+    },
     port: {
       type: 'string',
       short: 'p',
@@ -103,6 +106,7 @@ async function runBuild() {
   await build({
     config,
     env: process.env as any,
+    partial: !!values['experimental-partial'],
     deploy:
       (values['with-vercel'] ?? !!process.env.VERCEL
         ? values['with-vercel-static']

packages/waku/src/lib/builder/build.ts

@@ -165,6 +165,7 @@ const buildServerBundle = async (
     | 'aws-lambda'
     | false,
   isNodeCompatible: boolean,
+  partial: boolean,
 ) => {
   const serverBuildOutput = await buildVite({
     plugins: [
@@ -247,6 +248,7 @@ const buildServerBundle = async (
     },
     publicDir: false,
     build: {
+      emptyOutDir: !partial,
       ssr: true,
       ssrEmitAssets: true,
       target: 'node18',
@@ -275,6 +277,7 @@ const buildSsrBundle = async (
   clientEntryFiles: Record<string, string>,
   serverBuildOutput: Awaited<ReturnType<typeof buildServerBundle>>,
   isNodeCompatible: boolean,
+  partial: boolean,
 ) => {
   const cssAssets = serverBuildOutput.output.flatMap(({ type, fileName }) =>
     type === 'asset' && fileName.endsWith('.css') ? [fileName] : [],
@@ -310,6 +313,7 @@ const buildSsrBundle = async (
     },
     publicDir: false,
     build: {
+      emptyOutDir: !partial,
       ssr: true,
       target: 'node18',
       outDir: joinPath(rootDir, config.distDir, DIST_SSR),
@@ -343,6 +347,7 @@ const buildClientBundle = async (
   config: ResolvedConfig,
   clientEntryFiles: Record<string, string>,
   serverBuildOutput: Awaited<ReturnType<typeof buildServerBundle>>,
+  partial: boolean,
 ) => {
   const nonJsAssets = serverBuildOutput.output.flatMap(({ type, fileName }) =>
     type === 'asset' && !fileName.endsWith('.js') ? [fileName] : [],
@@ -361,6 +366,7 @@ const buildClientBundle = async (
       rscManagedPlugin({ ...config, addMainToInput: true }),
     ],
     build: {
+      emptyOutDir: !partial,
       outDir: joinPath(rootDir, config.distDir, DIST_PUBLIC),
       rollupOptions: {
         onwarn,
@@ -426,6 +432,10 @@ const emitRscFiles = async (
           config.rscPath,
           encodeInput(input),
         );
+        // Skip if the file already exists.
+        if (existsSync(destRscFile)) {
+          continue;
+        }
         await mkdir(joinPath(destRscFile, '..'), { recursive: true });
         const readable = await renderRsc(
           {
@@ -549,6 +559,10 @@ const emitHtmlFiles = async (
               ? '404.html' // HACK special treatment for 404, better way?
               : pathname + '/index.html',
         );
+        // In partial mode, skip if the file already exists.
+        if (existsSync(destHtmlFile)) {
+          return;
+        }
         const htmlReadable = await renderHtml({
           config,
           pathname,
@@ -606,6 +620,7 @@ export const publicIndexHtml = ${JSON.stringify(publicIndexHtml)};
 export async function build(options: {
   config: Config;
   env?: Record<string, string>;
+  partial?: boolean;
   deploy?:
     | 'vercel-static'
     | 'vercel-serverless'
@@ -643,19 +658,22 @@ export async function build(options: {
       (options.deploy === 'deno' ? 'deno' : false) ||
       (options.deploy === 'aws-lambda' ? 'aws-lambda' : false),
     isNodeCompatible,
+    !!options.partial,
   );
   await buildSsrBundle(
     rootDir,
     config,
     clientEntryFiles,
     serverBuildOutput,
     isNodeCompatible,
+    !!options.partial,
   );
   const clientBuildOutput = await buildClientBundle(
     rootDir,
     config,
     clientEntryFiles,
     serverBuildOutput,
+    !!options.partial,
   );
 
   const distEntries = await import(filePathToFileURL(distEntriesFile));

tsconfig.e2e.json

@@ -32,6 +32,9 @@
     },
     {
       "path": "./e2e/fixtures/ssg-performance/tsconfig.json"
+    },
+    {
+      "path": "./e2e/fixtures/partial-build/tsconfig.json"
     }
   ]
 }