build: run playwright browsers in a container for visual regression testing

.github/workflows/continuous-integration.yml

@@ -175,10 +175,14 @@ jobs:
 
       - name: Install browser dependencies
         run: yarn playwright install-deps
+        # This step must not fail, so we define a fallback, which will succeed, even if
+        # the visual regression tests failed. This will be evaluated in the secure workflow.
       - name: Run visual regression tests
-        run: yarn test:visual-regression
+        run: yarn test:visual-regression || true
         env:
           NODE_ENV: production
+      - name: Prepare failed screenshot artifacts
+        run: yarn ts-hooks tools/visual-regression-testing/prepare-failed-screenshots-artifact.ts
       - name: Store visual regression output
         uses: actions/upload-artifact@v4
         with:

docs/DEVELOPER.md

@@ -38,15 +38,15 @@ rendering.
 
 It is possible to debug tests and/or run them in isolation with Visual Studio Code.
 The following code snippet can be placed in `.vscode/launch.json`.
-Replace `test:csr` with either `test:ssr:hydrated` or `test:ssr:non-hydrated` to test SSR.
+Replace `test:csr` with `test:ssr` to test SSR.
 Add the `--debug` param to enable breakpoint debugging and the detailed test report.
 
 ```json
   ...
   {
     "name": "Test",
     "request": "launch",
-    "runtimeArgs": ["test:csr", "${relativeFile}", "--watch"],
+    "runtimeArgs": ["test:csr", "--file=${relativeFile}", "--watch"],
     "runtimeExecutable": "yarn",
     "skipFiles": ["<node_internals>/**"],
     "type": "node",
@@ -60,11 +60,11 @@ Add the `--debug` param to enable breakpoint debugging and the detailed test rep
 It is possible to debug tests and/or run them in isolation also with IntelliJ IDEA.
 From the title bar, open the 'Run' menu, then select 'Edit configuration'.
 Create and save a new `npm` configuration with the following parameters,
-possibly replacing `test:csr` with either `test:ssr:hydrated` or `test:ssr:non-hydrated` to test SSR:
+possibly replacing `test:csr` with `test:ssr` to test SSR:
 
 - Command: `run`
 - Scripts: `test:csr`
-- Arguments: `**/$FileName$ --watch`
+- Arguments: `--file=**/$FileName$ --watch`
 
 Finally, open the file you want to test and run the script.
 Add the `--debug` param to enable breakpoint debugging and the detailed test report.

package.json

@@ -63,7 +63,8 @@
     "test:snapshot": "yarn test:csr --ci --update-snapshots",
     "test:csr": "yarn -s wtr --group default",
     "test:ssr": "yarn -s wtr --group ssr",
-    "test:visual-regression": "yarn -s ts-hooks tools/visual-regression-testing/exec.ts --config=web-test-runner.config.ts --group=visual-regression --all-browsers",
+    "test:visual-regression": "yarn -s wtr --group visual-regression --all-browsers",
+    "test:playwright-server": "yarn -s ts-hooks tools/web-test-runner/start-playwright-server.ts",
     "wtr": "yarn -s ts-hooks node_modules/@web/test-runner/dist/bin.js --config=web-test-runner.config.ts",
     "ts-hooks": "node --no-deprecation --enable-source-maps --import ./tools/node-esm-hook/register-hooks.js",
     "prepare": "husky"

tools/visual-regression-testing/prepare-failed-screenshots-artifact.ts

@@ -0,0 +1,30 @@
+import { cpSync, existsSync, mkdirSync, writeFileSync, type CopySyncOptions } from 'node:fs';
+
+import * as glob from 'glob';
+
+// When visual regression tests have failed, we only want to pack the relevant screenshots
+// into the artifact transfered to the secure workflow, as uploading and downloading the full
+// baseline would take far longer.
+// Due to this we copy the necessary screenshots to /dist/screenshots-artifact which will
+// be moved to /dist/screenshots in the secure workflow.
+
+const screenshotDir = new URL('../../dist/screenshots/', import.meta.url);
+const artifactDir = new URL('../../dist/screenshots-artifact/', import.meta.url);
+const copyOptions: CopySyncOptions = { force: true, recursive: true };
+mkdirSync(artifactDir, { recursive: true });
+writeFileSync(new URL('./.keep', artifactDir), '', 'utf8');
+
+const failedDirs = glob.sync('*/failed/', { cwd: screenshotDir });
+for (const failedDir of failedDirs.map((d) => `./${d}`)) {
+  cpSync(new URL(failedDir, screenshotDir), new URL(failedDir, artifactDir), copyOptions);
+}
+
+const failedFiles = glob
+  .sync('*/failed/**/*.png', { cwd: artifactDir, ignore: '**/*-diff.png' })
+  .map((p) => p.replace('/failed/', '/baseline/'));
+for (const failedFile of failedFiles.map((f) => `./${f}`)) {
+  const baselineFile = new URL(failedFile, screenshotDir);
+  if (existsSync(baselineFile)) {
+    cpSync(baselineFile, new URL(failedFile, artifactDir), copyOptions);
+  }
+}

tools/web-test-runner/configure-container-playwright-browser.ts

@@ -0,0 +1,56 @@
+import type { PlaywrightLauncher, ProductType } from '@web/test-runner-playwright';
+import type { Browser, LaunchOptions } from 'playwright';
+import * as playwright from 'playwright';
+
+import { playwrightWebsocketAddress } from './container-playwright-browser-plugin.js';
+
+interface PlaywrightLauncherPrivate {
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  __connectBrowserPromise: Promise<Browser> | undefined;
+  browser: Browser | undefined;
+  product: ProductType;
+  launchOptions: LaunchOptions;
+}
+
+/**
+ * Configure playwright browser execution to run the browsers in a container.
+ *
+ * @see https://github.com/microsoft/playwright/issues/26482
+ * @param browser The playwright browser launcher.
+ */
+export function configureRemotePlaywrightBrowser(browser: PlaywrightLauncher): void {
+  // The original implementation calls launch instead of connect,
+  // so we overwrite the original method with this variant which
+  // calls connect with the websocket endpoint.
+  async function getOrStartBrowser(this: PlaywrightLauncherPrivate): Promise<Browser> {
+    if (this.__connectBrowserPromise) {
+      return this.__connectBrowserPromise;
+    }
+
+    if (!this.browser || !this.browser?.isConnected()) {
+      this.__connectBrowserPromise = (async () => {
+        // eslint-disable-next-line import-x/namespace
+        const browser = await playwright[this.product].connect(playwrightWebsocketAddress, {
+          headers: { 'x-playwright-launch-options': JSON.stringify(this.launchOptions) },
+        });
+        return browser;
+      })();
+      const browser = await this.__connectBrowserPromise;
+      this.browser = browser;
+      this.__connectBrowserPromise = undefined;
+    }
+    return this.browser;
+  }
+  (browser as any).getOrStartBrowser = getOrStartBrowser;
+
+  // Inside the container the dev server from `@web/test-runner` is not available
+  // so we need to adapt the address to point to the host machine.
+  const startSession = browser.startSession;
+  browser.startSession = async function (sessionId: string, url: string): Promise<void> {
+    await startSession.call(
+      this,
+      sessionId,
+      url.replace('http://localhost:', 'http://host.containers.internal:'),
+    );
+  };
+}

tools/web-test-runner/container-playwright-browser-plugin.ts

@@ -0,0 +1,85 @@
+import { execSync, spawn } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { platform } from 'node:os';
+
+import type { TestRunnerPlugin } from '@web/test-runner';
+
+function executableIsAvailable(name: string): string | null {
+  try {
+    execSync(`${platform().startsWith('win') ? 'where' : 'which'} ${name}`, { encoding: 'utf8' });
+    return name;
+  } catch (error) {
+    return null;
+  }
+}
+
+const containerCmd =
+  process.env.CONTAINER_CMD ?? executableIsAvailable('podman') ?? executableIsAvailable('docker');
+const pkgJson = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8'));
+const playwrightVersion = pkgJson.devDependencies.playwright;
+const port = 3000;
+export const playwrightWebsocketAddress = `ws://localhost:${port}`;
+// See https://github.com/microsoft/playwright/issues/26482
+export const startPlaywrightServerCommand = [
+  containerCmd ?? 'docker',
+  'run',
+  '-p',
+  `${port}:${port}`,
+  '--rm',
+  '--init',
+  '--workdir=/home/pwuser',
+  '--entrypoint=/bin/sh',
+  `mcr.microsoft.com/playwright:v${playwrightVersion}-jammy`,
+  `-c`,
+  `npx -y playwright@${playwrightVersion} run-server --port ${port} --host 0.0.0.0`,
+];
+
+// Reference: https://github.com/remcovaes/web-test-runner-vite-plugin
+export function containerPlaywrightBrowserPlugin(): TestRunnerPlugin {
+  let abortController: AbortController;
+
+  if (!containerCmd) {
+    console.log('Either docker or podman need to be installed!');
+    process.exit(1);
+  }
+
+  return {
+    name: 'remote-playwright-browser-plugin',
+
+    async serverStart({ logger }) {
+      logger.log('Starting playwright browsers in a container');
+      abortController = new AbortController();
+      await new Promise<void>((resolve, reject) => {
+        let id: NodeJS.Timeout | undefined = undefined;
+        spawn(startPlaywrightServerCommand[0], startPlaywrightServerCommand.slice(1), {
+          signal: abortController.signal,
+        }).on('error', (err) => {
+          clearInterval(id);
+          reject(err);
+        });
+        id = setInterval(() => {
+          const timeout = AbortSignal.timeout(950);
+          const ws = new WebSocket(playwrightWebsocketAddress);
+          timeout.addEventListener('abort', () => ws.close());
+          ws.addEventListener(
+            'open',
+            () => {
+              console.log('Playwright container is ready');
+              ws.close();
+              clearInterval(id);
+              resolve();
+            },
+            { signal: timeout },
+          );
+        }, 1000);
+        AbortSignal.timeout(60000).addEventListener('abort', () => {
+          clearInterval(id);
+          reject('Failed to start container');
+        });
+      });
+    },
+    async serverStop() {
+      abortController.abort();
+    },
+  };
+}

tools/web-test-runner/index.ts

@@ -1,3 +1,5 @@
+export * from './configure-container-playwright-browser.js';
+export * from './container-playwright-browser-plugin.js';
 export * from './minimal-reporter.js';
 export * from './patched-summary-reporter.js';
 export * from './visual-regression-plugin-config.js';

tools/web-test-runner/start-playwright-server.ts

@@ -0,0 +1,14 @@
+import { spawn } from 'node:child_process';
+
+import { startPlaywrightServerCommand } from './container-playwright-browser-plugin.js';
+
+const abortController = new AbortController();
+spawn(startPlaywrightServerCommand[0], startPlaywrightServerCommand.slice(1), {
+  stdio: 'inherit',
+  signal: abortController.signal,
+});
+
+process.on('SIGINT', () => {
+  abortController.abort();
+  process.exit(0);
+});