From e3b4bde0e6a62463d868537f734f80e8644ce0bd Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Thu, 13 Jun 2024 20:59:09 +0200 Subject: [PATCH 1/5] build: run playwright browsers in a container for visual regression testing --- .github/workflows/continuous-integration.yml | 2 + package.json | 3 +- tools/visual-regression-testing/exec.ts | 105 ------------------ .../prepare-failed-screenshots-artifact.ts | 30 +++++ .../testing.Dockerfile | 7 -- .../testing.Dockerfile.dockerignore | 3 - .../configure-container-playwright-browser.ts | 57 ++++++++++ .../container-playwright-browser-plugin.ts | 86 ++++++++++++++ tools/web-test-runner/index.ts | 2 + .../start-playwright-server.ts | 14 +++ web-test-runner.config.ts | 12 +- 11 files changed, 204 insertions(+), 117 deletions(-) delete mode 100644 tools/visual-regression-testing/exec.ts create mode 100644 tools/visual-regression-testing/prepare-failed-screenshots-artifact.ts delete mode 100644 tools/visual-regression-testing/testing.Dockerfile delete mode 100644 tools/visual-regression-testing/testing.Dockerfile.dockerignore create mode 100644 tools/web-test-runner/configure-container-playwright-browser.ts create mode 100644 tools/web-test-runner/container-playwright-browser-plugin.ts create mode 100644 tools/web-test-runner/start-playwright-server.ts diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index d5664b017e..3c1b9ceaf0 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -179,6 +179,8 @@ jobs: run: yarn test:visual-regression env: NODE_ENV: production + - name: Prepare failed screenshot artifacts + run: yarn ts-hooks tools/visual-regression-testing/prepare-failed-screenshots-artifact.ts - name: Store visual regression output uses: actions/upload-artifact@v4 with: diff --git a/package.json b/package.json index 8be97c082c..bd230b761d 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,8 @@ "test:snapshot": "yarn test:csr --ci --update-snapshots", "test:csr": "yarn -s wtr --group default", "test:ssr": "yarn -s wtr --group ssr", - "test:visual-regression": "yarn -s ts-hooks tools/visual-regression-testing/exec.ts --config=web-test-runner.config.ts --group=visual-regression --all-browsers", + "test:visual-regression": "yarn -s wtr --group visual-regression --all-browsers --container", + "test:playwright-server": "yarn -s ts-hooks tools/web-test-runner/start-playwright-server.ts", "wtr": "yarn -s ts-hooks node_modules/@web/test-runner/dist/bin.js --config=web-test-runner.config.ts", "ts-hooks": "node --no-deprecation --enable-source-maps --import ./tools/node-esm-hook/register-hooks.js", "prepare": "husky" diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts deleted file mode 100644 index 0f95de0070..0000000000 --- a/tools/visual-regression-testing/exec.ts +++ /dev/null @@ -1,105 +0,0 @@ -// This script checks which OS the visual regression testing is run on -// and if it is not Linux, runs it in a container. - -import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; -import { cpSync, existsSync, mkdirSync, writeFileSync } from 'fs'; -import { platform } from 'os'; - -import { startTestRunner } from '@web/test-runner'; -import * as glob from 'glob'; - -if (process.env.GITHUB_ACTIONS) { - // When being run on GitHub Actions we have two use cases. - // Baseline generation for which our expectation is to not fail. - // Diff generation if any test fails, for which we copy only the necessary - // files to dist/screenshots-artifact/ to reduce artifact size. - const runner = await startTestRunner({ autoExitProcess: false }); - if (!runner) { - throw new Error( - `Unexpected state. Test runner not available. Check tools/visual-regression-testing/exec.ts execution.`, - ); - } - await new Promise((r) => runner.on('stopped', r)); - - const screenshotDir = new URL('../../dist/screenshots/', import.meta.url); - const artifactDir = new URL('../../dist/screenshots-artifact/', import.meta.url); - mkdirSync(artifactDir, { recursive: true }); - writeFileSync(new URL('./.keep', artifactDir), '', 'utf8'); - - if (runner.passed) { - // Tests passed. Do nothing. - process.exit(0); - } - - // When visual regression tests have failed, we only want to pack the relevant screenshots - // into the artifact transfered to the secure workflow, as uploading and downloading the full - // baseline would take far longer. - // Due to this we copy the necessary screenshots to /dist/screenshots-artifact which will - // be moved to /dist/screenshots in the secure workflow. - const failedDirs = glob.sync('*/failed/', { cwd: screenshotDir }); - for (const failedDir of failedDirs) { - cpSync(new URL(`./${failedDir}`, screenshotDir), new URL(`./${failedDir}`, artifactDir), { - force: true, - recursive: true, - }); - } - - const failedFiles = glob - .sync('*/failed/**/*.png', { cwd: artifactDir, ignore: '**/*-diff.png' }) - .map((p) => p.replace('/failed/', '/baseline/')); - for (const failedFile of failedFiles) { - const baselineFile = new URL(`./${failedFile}`, screenshotDir); - if (existsSync(baselineFile)) { - cpSync(baselineFile, new URL(`./${failedFile}`, artifactDir), { - force: true, - recursive: true, - }); - } - } -} else if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { - await startTestRunner(); -} else { - function executableIsAvailable(name: string): string | null { - try { - execSync(`${platform().startsWith('win') ? 'where' : 'which'} ${name}`, { encoding: 'utf8' }); - return name; - } catch (error) { - return null; - } - } - - const containerCmd = executableIsAvailable('podman') ?? executableIsAvailable('docker'); - if (!containerCmd) { - console.log('Either docker or podman need to be installed!'); - process.exit(1); - } - - const args = process.argv.slice(2); - const cwd = new URL('../../', import.meta.url); - const tag = 'lyne-vrt'; - const execOptions: ExecSyncOptionsWithStringEncoding = { - encoding: 'utf8', - stdio: 'inherit', - cwd, - }; - const branch = - process.env.GITHUB_REF_NAME ?? - process.env.BRANCH ?? - execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); - execSync( - `${containerCmd} build ` + - '--file=tools/visual-regression-testing/testing.Dockerfile ' + - `--tag=${tag} .`, - execOptions, - ); - console.log(`\nTest image ready\n`); - mkdirSync(new URL('./dist/screenshots', cwd), { recursive: true }); - execSync( - `${containerCmd} run -it --rm --ipc=host ` + - `--env=BRANCH="${branch}"` + - `--volume=./dist/screenshots:/dist/screenshots ` + - `--entrypoint='["yarn", "wtr"${args.map((a) => `, "${a}"`).join('')}]' ` + - tag, - execOptions, - ); -} diff --git a/tools/visual-regression-testing/prepare-failed-screenshots-artifact.ts b/tools/visual-regression-testing/prepare-failed-screenshots-artifact.ts new file mode 100644 index 0000000000..0e4ae70fbd --- /dev/null +++ b/tools/visual-regression-testing/prepare-failed-screenshots-artifact.ts @@ -0,0 +1,30 @@ +import { cpSync, existsSync, mkdirSync, writeFileSync, type CopySyncOptions } from 'node:fs'; + +import * as glob from 'glob'; + +// When visual regression tests have failed, we only want to pack the relevant screenshots +// into the artifact transfered to the secure workflow, as uploading and downloading the full +// baseline would take far longer. +// Due to this we copy the necessary screenshots to /dist/screenshots-artifact which will +// be moved to /dist/screenshots in the secure workflow. + +const screenshotDir = new URL('../../dist/screenshots/', import.meta.url); +const artifactDir = new URL('../../dist/screenshots-artifact/', import.meta.url); +const copyOptions: CopySyncOptions = { force: true, recursive: true }; +mkdirSync(artifactDir, { recursive: true }); +writeFileSync(new URL('./.keep', artifactDir), '', 'utf8'); + +const failedDirs = glob.sync('*/failed/', { cwd: screenshotDir }); +for (const failedDir of failedDirs.map((d) => `./${d}`)) { + cpSync(new URL(failedDir, screenshotDir), new URL(failedDir, artifactDir), copyOptions); +} + +const failedFiles = glob + .sync('*/failed/**/*.png', { cwd: artifactDir, ignore: '**/*-diff.png' }) + .map((p) => p.replace('/failed/', '/baseline/')); +for (const failedFile of failedFiles.map((f) => `./${f}`)) { + const baselineFile = new URL(failedFile, screenshotDir); + if (existsSync(baselineFile)) { + cpSync(baselineFile, new URL(failedFile, artifactDir), copyOptions); + } +} diff --git a/tools/visual-regression-testing/testing.Dockerfile b/tools/visual-regression-testing/testing.Dockerfile deleted file mode 100644 index f56f1eb533..0000000000 --- a/tools/visual-regression-testing/testing.Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM mcr.microsoft.com/playwright:v1.44.0-jammy - -COPY package.json ./ -COPY yarn.lock ./ -RUN yarn install --frozen-lockfile --non-interactive - -COPY . . diff --git a/tools/visual-regression-testing/testing.Dockerfile.dockerignore b/tools/visual-regression-testing/testing.Dockerfile.dockerignore deleted file mode 100644 index 5de02b1469..0000000000 --- a/tools/visual-regression-testing/testing.Dockerfile.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -coverage -dist -node_modules \ No newline at end of file diff --git a/tools/web-test-runner/configure-container-playwright-browser.ts b/tools/web-test-runner/configure-container-playwright-browser.ts new file mode 100644 index 0000000000..5c284c718a --- /dev/null +++ b/tools/web-test-runner/configure-container-playwright-browser.ts @@ -0,0 +1,57 @@ +import type { PlaywrightLauncher, ProductType } from '@web/test-runner-playwright'; +import type { Browser, LaunchOptions } from 'playwright'; +import * as playwright from 'playwright'; + +import { playwrightWebsocketAddress } from './container-playwright-browser-plugin.js'; + +interface PlaywrightLauncherPrivate { + // eslint-disable-next-line @typescript-eslint/naming-convention + __connectBrowserPromise: Promise | undefined; + browser: Browser | undefined; + product: ProductType; + launchOptions: LaunchOptions; +} + +/** + * Configure playwright browser execution to run the browsers in a container. + * + * @see https://github.com/microsoft/playwright/issues/26482 + * @param browser The playwright browser launcher. + */ +export function configureRemotePlaywrightBrowser(browser: PlaywrightLauncher): void { + // The original implementation calls launch instead of connect, + // so we overwrite the original method with this variant which + // calls connect with the websocket endpoint. + async function getOrStartBrowser(this: PlaywrightLauncherPrivate): Promise { + if (this.__connectBrowserPromise) { + return this.__connectBrowserPromise; + } + + if (!this.browser || !this.browser?.isConnected()) { + this.__connectBrowserPromise = (async () => { + // eslint-disable-next-line import-x/namespace + const browser = await playwright[this.product].connect( + playwrightWebsocketAddress, + this.launchOptions, + ); + return browser; + })(); + const browser = await this.__connectBrowserPromise; + this.browser = browser; + this.__connectBrowserPromise = undefined; + } + return this.browser; + } + (browser as any).getOrStartBrowser = getOrStartBrowser; + + // Inside the container the dev server from `@web/test-runner` is not available + // so we need to adapt the address to point to the host machine. + const startSession = browser.startSession; + browser.startSession = async function (sessionId: string, url: string): Promise { + await startSession.call( + this, + sessionId, + url.replace('http://localhost:', 'http://hostmachine:'), + ); + }; +} diff --git a/tools/web-test-runner/container-playwright-browser-plugin.ts b/tools/web-test-runner/container-playwright-browser-plugin.ts new file mode 100644 index 0000000000..76f159ba61 --- /dev/null +++ b/tools/web-test-runner/container-playwright-browser-plugin.ts @@ -0,0 +1,86 @@ +import { execSync, spawn } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { platform } from 'node:os'; + +import type { TestRunnerPlugin } from '@web/test-runner'; + +function executableIsAvailable(name: string): string | null { + try { + execSync(`${platform().startsWith('win') ? 'where' : 'which'} ${name}`, { encoding: 'utf8' }); + return name; + } catch (error) { + return null; + } +} + +const containerCmd = + process.env.CONTAINER_CMD ?? executableIsAvailable('podman') ?? executableIsAvailable('docker'); +const pkgJson = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')); +const playwrightVersion = pkgJson.devDependencies.playwright; +const port = 3000; +export const playwrightWebsocketAddress = `ws://localhost:${port}`; +// See https://github.com/microsoft/playwright/issues/26482 +export const startPlaywrightServerCommand = [ + containerCmd ?? 'docker', + 'run', + '-p', + `${port}:${port}`, + '--add-host=hostmachine:host-gateway', + '--rm', + '--init', + '--workdir=/home/pwuser', + '--entrypoint=/bin/sh', + `mcr.microsoft.com/playwright:v${playwrightVersion}-jammy`, + `-c`, + `npx -y playwright@${playwrightVersion} run-server --port ${port} --host 0.0.0.0`, +]; + +// Reference: https://github.com/remcovaes/web-test-runner-vite-plugin +export function containerPlaywrightBrowserPlugin(): TestRunnerPlugin { + let abortController: AbortController; + + if (!containerCmd) { + console.log('Either docker or podman need to be installed!'); + process.exit(1); + } + + return { + name: 'remote-playwright-browser-plugin', + + async serverStart({ logger }) { + logger.log('Starting playwright browsers in a container'); + abortController = new AbortController(); + await new Promise((resolve, reject) => { + let id: NodeJS.Timeout | undefined = undefined; + spawn(startPlaywrightServerCommand[0], startPlaywrightServerCommand.slice(1), { + signal: abortController.signal, + }).on('error', (err) => { + clearInterval(id); + reject(err); + }); + id = setInterval(() => { + const timeout = AbortSignal.timeout(950); + const ws = new WebSocket(playwrightWebsocketAddress); + timeout.addEventListener('abort', () => ws.close()); + ws.addEventListener( + 'open', + () => { + console.log('Playwright container is ready'); + ws.close(); + clearInterval(id); + resolve(); + }, + { signal: timeout }, + ); + }, 1000); + AbortSignal.timeout(60000).addEventListener('abort', () => { + clearInterval(id); + reject('Failed to start container'); + }); + }); + }, + async serverStop() { + abortController.abort(); + }, + }; +} diff --git a/tools/web-test-runner/index.ts b/tools/web-test-runner/index.ts index b2cced322b..f15f5ed9bc 100644 --- a/tools/web-test-runner/index.ts +++ b/tools/web-test-runner/index.ts @@ -1,3 +1,5 @@ +export * from './configure-container-playwright-browser.js'; +export * from './container-playwright-browser-plugin.js'; export * from './minimal-reporter.js'; export * from './patched-summary-reporter.js'; export * from './visual-regression-plugin-config.js'; diff --git a/tools/web-test-runner/start-playwright-server.ts b/tools/web-test-runner/start-playwright-server.ts new file mode 100644 index 0000000000..fdda2fadbc --- /dev/null +++ b/tools/web-test-runner/start-playwright-server.ts @@ -0,0 +1,14 @@ +import { spawn } from 'node:child_process'; + +import { startPlaywrightServerCommand } from './container-playwright-browser-plugin.js'; + +const abortController = new AbortController(); +spawn(startPlaywrightServerCommand[0], startPlaywrightServerCommand.slice(1), { + stdio: 'inherit', + signal: abortController.signal, +}); + +process.on('SIGINT', () => { + abortController.abort(); + process.exit(0); +}); diff --git a/web-test-runner.config.ts b/web-test-runner.config.ts index 2e544abfc8..9e8f3811d3 100644 --- a/web-test-runner.config.ts +++ b/web-test-runner.config.ts @@ -8,14 +8,16 @@ import { type TestRunnerGroupConfig, } from '@web/test-runner'; import { a11ySnapshotPlugin } from '@web/test-runner-commands/plugins'; -import { type PlaywrightLauncherArgs, playwrightLauncher } from '@web/test-runner-playwright'; +import { type PlaywrightLauncherArgs, playwrightLauncher, type PlaywrightLauncher } from '@web/test-runner-playwright'; import { puppeteerLauncher } from '@web/test-runner-puppeteer'; import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin'; import { initCompiler } from 'sass'; import { + configureRemotePlaywrightBrowser, minimalReporter, patchedSummaryReporter, + containerPlaywrightBrowserPlugin, visualRegressionConfig, vitePlugin, } from './tools/web-test-runner/index.js'; @@ -32,6 +34,7 @@ const { values: cliArgs } = parseArgs({ 'update-visual-baseline': { type: 'boolean' }, group: { type: 'string' }, ssr: { type: 'boolean' }, + container: { type: 'boolean' }, }, }); @@ -122,6 +125,12 @@ if (cliArgs.group === 'visual-regression') { groups.push({ name: 'visual-regression', files: 'src/**/*.visual.spec.ts', testRunnerHtml }); } +if (cliArgs.container) { + browsers + .filter((b): b is PlaywrightLauncher => b.type === 'playwright') + .forEach((browser) => configureRemotePlaywrightBrowser(browser)); +} + export default { files: ['src/**/*.spec.ts', '!**/*.{visual,ssr}.spec.ts'], groups, @@ -140,6 +149,7 @@ export default { ...visualRegressionConfig, update: !!cliArgs['update-visual-baseline'], }), + ...(cliArgs.container ? [containerPlaywrightBrowserPlugin()] : []), ], testFramework: { config: { From 415436c2f7ff6b89b50d18f404a909165ddf644a Mon Sep 17 00:00:00 2001 From: Jeremias Peier Date: Fri, 14 Jun 2024 16:06:07 +0200 Subject: [PATCH 2/5] fix: let it really run --- .../configure-container-playwright-browser.ts | 9 ++++----- .../container-playwright-browser-plugin.ts | 1 - 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tools/web-test-runner/configure-container-playwright-browser.ts b/tools/web-test-runner/configure-container-playwright-browser.ts index 5c284c718a..9f3067de20 100644 --- a/tools/web-test-runner/configure-container-playwright-browser.ts +++ b/tools/web-test-runner/configure-container-playwright-browser.ts @@ -30,10 +30,9 @@ export function configureRemotePlaywrightBrowser(browser: PlaywrightLauncher): v if (!this.browser || !this.browser?.isConnected()) { this.__connectBrowserPromise = (async () => { // eslint-disable-next-line import-x/namespace - const browser = await playwright[this.product].connect( - playwrightWebsocketAddress, - this.launchOptions, - ); + const browser = await playwright[this.product].connect(playwrightWebsocketAddress, { + headers: { 'x-playwright-launch-options': JSON.stringify(this.launchOptions) }, + }); return browser; })(); const browser = await this.__connectBrowserPromise; @@ -51,7 +50,7 @@ export function configureRemotePlaywrightBrowser(browser: PlaywrightLauncher): v await startSession.call( this, sessionId, - url.replace('http://localhost:', 'http://hostmachine:'), + url.replace('http://localhost:', 'http://host.containers.internal:'), ); }; } diff --git a/tools/web-test-runner/container-playwright-browser-plugin.ts b/tools/web-test-runner/container-playwright-browser-plugin.ts index 76f159ba61..8693d4e5ab 100644 --- a/tools/web-test-runner/container-playwright-browser-plugin.ts +++ b/tools/web-test-runner/container-playwright-browser-plugin.ts @@ -25,7 +25,6 @@ export const startPlaywrightServerCommand = [ 'run', '-p', `${port}:${port}`, - '--add-host=hostmachine:host-gateway', '--rm', '--init', '--workdir=/home/pwuser', From a0c6f275c28b535840ffa36117819a2c0e5d5b64 Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Tue, 25 Jun 2024 11:22:29 +0200 Subject: [PATCH 3/5] fix: wip --- package.json | 2 +- web-test-runner.config.ts | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index bd230b761d..bc7d9a2ea4 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "test:snapshot": "yarn test:csr --ci --update-snapshots", "test:csr": "yarn -s wtr --group default", "test:ssr": "yarn -s wtr --group ssr", - "test:visual-regression": "yarn -s wtr --group visual-regression --all-browsers --container", + "test:visual-regression": "yarn -s wtr --group visual-regression --all-browsers", "test:playwright-server": "yarn -s ts-hooks tools/web-test-runner/start-playwright-server.ts", "wtr": "yarn -s ts-hooks node_modules/@web/test-runner/dist/bin.js --config=web-test-runner.config.ts", "ts-hooks": "node --no-deprecation --enable-source-maps --import ./tools/node-esm-hook/register-hooks.js", diff --git a/web-test-runner.config.ts b/web-test-runner.config.ts index 9e8f3811d3..6b2a1960bd 100644 --- a/web-test-runner.config.ts +++ b/web-test-runner.config.ts @@ -1,3 +1,4 @@ +import { platform } from 'node:os'; import { parseArgs } from 'node:util'; import { litSsrPlugin } from '@lit-labs/testing/web-test-runner-ssr-plugin.js'; @@ -8,7 +9,11 @@ import { type TestRunnerGroupConfig, } from '@web/test-runner'; import { a11ySnapshotPlugin } from '@web/test-runner-commands/plugins'; -import { type PlaywrightLauncherArgs, playwrightLauncher, type PlaywrightLauncher } from '@web/test-runner-playwright'; +import { + type PlaywrightLauncherArgs, + playwrightLauncher, + type PlaywrightLauncher, +} from '@web/test-runner-playwright'; import { puppeteerLauncher } from '@web/test-runner-puppeteer'; import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin'; import { initCompiler } from 'sass'; @@ -35,6 +40,7 @@ const { values: cliArgs } = parseArgs({ group: { type: 'string' }, ssr: { type: 'boolean' }, container: { type: 'boolean' }, + local: { type: 'boolean' }, }, }); @@ -123,6 +129,12 @@ const groups: TestRunnerGroupConfig[] = [ // The visual regression test group is only added when explicitly set, as the tests are very expensive. if (cliArgs.group === 'visual-regression') { groups.push({ name: 'visual-regression', files: 'src/**/*.visual.spec.ts', testRunnerHtml }); + if (!cliArgs.local && platform() !== 'linux') { + console.log( + `Running visual regression tests in a non-linux environment. Switching to container usage. Use --local to opt-out.`, + ); + cliArgs.container = true; + } } if (cliArgs.container) { From b471bbcfe3cb1de76e44ddbd9d63ee655b9826d3 Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Tue, 25 Jun 2024 12:08:32 +0200 Subject: [PATCH 4/5] fix: review --- docs/DEVELOPER.md | 8 ++++---- web-test-runner.config.ts | 12 +++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/DEVELOPER.md b/docs/DEVELOPER.md index a8def72b76..98faa16c46 100644 --- a/docs/DEVELOPER.md +++ b/docs/DEVELOPER.md @@ -38,7 +38,7 @@ rendering. It is possible to debug tests and/or run them in isolation with Visual Studio Code. The following code snippet can be placed in `.vscode/launch.json`. -Replace `test:csr` with either `test:ssr:hydrated` or `test:ssr:non-hydrated` to test SSR. +Replace `test:csr` with `test:ssr` to test SSR. Add the `--debug` param to enable breakpoint debugging and the detailed test report. ```json @@ -46,7 +46,7 @@ Add the `--debug` param to enable breakpoint debugging and the detailed test rep { "name": "Test", "request": "launch", - "runtimeArgs": ["test:csr", "${relativeFile}", "--watch"], + "runtimeArgs": ["test:csr", "--file=${relativeFile}", "--watch"], "runtimeExecutable": "yarn", "skipFiles": ["/**"], "type": "node", @@ -60,11 +60,11 @@ Add the `--debug` param to enable breakpoint debugging and the detailed test rep It is possible to debug tests and/or run them in isolation also with IntelliJ IDEA. From the title bar, open the 'Run' menu, then select 'Edit configuration'. Create and save a new `npm` configuration with the following parameters, -possibly replacing `test:csr` with either `test:ssr:hydrated` or `test:ssr:non-hydrated` to test SSR: +possibly replacing `test:csr` with `test:ssr` to test SSR: - Command: `run` - Scripts: `test:csr` -- Arguments: `**/$FileName$ --watch` +- Arguments: `--file=**/$FileName$ --watch` Finally, open the file you want to test and run the script. Add the `--debug` param to enable breakpoint debugging and the detailed test report. diff --git a/web-test-runner.config.ts b/web-test-runner.config.ts index 6b2a1960bd..b0771355ab 100644 --- a/web-test-runner.config.ts +++ b/web-test-runner.config.ts @@ -30,6 +30,7 @@ import { const { values: cliArgs } = parseArgs({ strict: false, options: { + file: { type: 'string' }, ci: { type: 'boolean', default: !!process.env.CI }, debug: { type: 'boolean' }, 'all-browsers': { type: 'boolean', short: 'a' }, @@ -122,13 +123,18 @@ const suppressedLogs = [ '[vite] connected.', ]; +const testFile = typeof cliArgs.file === 'string' && cliArgs.file ? cliArgs.file : undefined; const groups: TestRunnerGroupConfig[] = [ - { name: 'ssr', files: 'src/**/*.ssr.spec.ts', testRunnerHtml }, + { name: 'ssr', files: testFile ?? 'src/**/*.ssr.spec.ts', testRunnerHtml }, ]; // The visual regression test group is only added when explicitly set, as the tests are very expensive. if (cliArgs.group === 'visual-regression') { - groups.push({ name: 'visual-regression', files: 'src/**/*.visual.spec.ts', testRunnerHtml }); + groups.push({ + name: 'visual-regression', + files: testFile ?? 'src/**/*.visual.spec.ts', + testRunnerHtml, + }); if (!cliArgs.local && platform() !== 'linux') { console.log( `Running visual regression tests in a non-linux environment. Switching to container usage. Use --local to opt-out.`, @@ -144,7 +150,7 @@ if (cliArgs.container) { } export default { - files: ['src/**/*.spec.ts', '!**/*.{visual,ssr}.spec.ts'], + files: testFile ?? ['src/**/*.spec.ts', '!**/*.{visual,ssr}.spec.ts'], groups, nodeResolve: true, reporters: From 37b740f04296c2342d7c9bd84289ad3d14e8c337 Mon Sep 17 00:00:00 2001 From: Lukas Spirig Date: Tue, 25 Jun 2024 14:07:06 +0200 Subject: [PATCH 5/5] fix: review --- .github/workflows/continuous-integration.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 3c1b9ceaf0..9f3427a0ad 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -175,8 +175,10 @@ jobs: - name: Install browser dependencies run: yarn playwright install-deps + # This step must not fail, so we define a fallback, which will succeed, even if + # the visual regression tests failed. This will be evaluated in the secure workflow. - name: Run visual regression tests - run: yarn test:visual-regression + run: yarn test:visual-regression || true env: NODE_ENV: production - name: Prepare failed screenshot artifacts