diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index d5664b017e..9f3427a0ad 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -175,10 +175,14 @@ jobs: - name: Install browser dependencies run: yarn playwright install-deps + # This step must not fail, so we define a fallback, which will succeed, even if + # the visual regression tests failed. This will be evaluated in the secure workflow. - name: Run visual regression tests - run: yarn test:visual-regression + run: yarn test:visual-regression || true env: NODE_ENV: production + - name: Prepare failed screenshot artifacts + run: yarn ts-hooks tools/visual-regression-testing/prepare-failed-screenshots-artifact.ts - name: Store visual regression output uses: actions/upload-artifact@v4 with: diff --git a/docs/DEVELOPER.md b/docs/DEVELOPER.md index a8def72b76..98faa16c46 100644 --- a/docs/DEVELOPER.md +++ b/docs/DEVELOPER.md @@ -38,7 +38,7 @@ rendering. It is possible to debug tests and/or run them in isolation with Visual Studio Code. The following code snippet can be placed in `.vscode/launch.json`. -Replace `test:csr` with either `test:ssr:hydrated` or `test:ssr:non-hydrated` to test SSR. +Replace `test:csr` with `test:ssr` to test SSR. Add the `--debug` param to enable breakpoint debugging and the detailed test report. ```json @@ -46,7 +46,7 @@ Add the `--debug` param to enable breakpoint debugging and the detailed test rep { "name": "Test", "request": "launch", - "runtimeArgs": ["test:csr", "${relativeFile}", "--watch"], + "runtimeArgs": ["test:csr", "--file=${relativeFile}", "--watch"], "runtimeExecutable": "yarn", "skipFiles": ["/**"], "type": "node", @@ -60,11 +60,11 @@ Add the `--debug` param to enable breakpoint debugging and the detailed test rep It is possible to debug tests and/or run them in isolation also with IntelliJ IDEA. From the title bar, open the 'Run' menu, then select 'Edit configuration'. Create and save a new `npm` configuration with the following parameters, -possibly replacing `test:csr` with either `test:ssr:hydrated` or `test:ssr:non-hydrated` to test SSR: +possibly replacing `test:csr` with `test:ssr` to test SSR: - Command: `run` - Scripts: `test:csr` -- Arguments: `**/$FileName$ --watch` +- Arguments: `--file=**/$FileName$ --watch` Finally, open the file you want to test and run the script. Add the `--debug` param to enable breakpoint debugging and the detailed test report. diff --git a/package.json b/package.json index 8be97c082c..bc7d9a2ea4 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,8 @@ "test:snapshot": "yarn test:csr --ci --update-snapshots", "test:csr": "yarn -s wtr --group default", "test:ssr": "yarn -s wtr --group ssr", - "test:visual-regression": "yarn -s ts-hooks tools/visual-regression-testing/exec.ts --config=web-test-runner.config.ts --group=visual-regression --all-browsers", + "test:visual-regression": "yarn -s wtr --group visual-regression --all-browsers", + "test:playwright-server": "yarn -s ts-hooks tools/web-test-runner/start-playwright-server.ts", "wtr": "yarn -s ts-hooks node_modules/@web/test-runner/dist/bin.js --config=web-test-runner.config.ts", "ts-hooks": "node --no-deprecation --enable-source-maps --import ./tools/node-esm-hook/register-hooks.js", "prepare": "husky" diff --git a/tools/visual-regression-testing/exec.ts b/tools/visual-regression-testing/exec.ts deleted file mode 100644 index 0f95de0070..0000000000 --- a/tools/visual-regression-testing/exec.ts +++ /dev/null @@ -1,105 +0,0 @@ -// This script checks which OS the visual regression testing is run on -// and if it is not Linux, runs it in a container. - -import { execSync, type ExecSyncOptionsWithStringEncoding } from 'child_process'; -import { cpSync, existsSync, mkdirSync, writeFileSync } from 'fs'; -import { platform } from 'os'; - -import { startTestRunner } from '@web/test-runner'; -import * as glob from 'glob'; - -if (process.env.GITHUB_ACTIONS) { - // When being run on GitHub Actions we have two use cases. - // Baseline generation for which our expectation is to not fail. - // Diff generation if any test fails, for which we copy only the necessary - // files to dist/screenshots-artifact/ to reduce artifact size. - const runner = await startTestRunner({ autoExitProcess: false }); - if (!runner) { - throw new Error( - `Unexpected state. Test runner not available. Check tools/visual-regression-testing/exec.ts execution.`, - ); - } - await new Promise((r) => runner.on('stopped', r)); - - const screenshotDir = new URL('../../dist/screenshots/', import.meta.url); - const artifactDir = new URL('../../dist/screenshots-artifact/', import.meta.url); - mkdirSync(artifactDir, { recursive: true }); - writeFileSync(new URL('./.keep', artifactDir), '', 'utf8'); - - if (runner.passed) { - // Tests passed. Do nothing. - process.exit(0); - } - - // When visual regression tests have failed, we only want to pack the relevant screenshots - // into the artifact transfered to the secure workflow, as uploading and downloading the full - // baseline would take far longer. - // Due to this we copy the necessary screenshots to /dist/screenshots-artifact which will - // be moved to /dist/screenshots in the secure workflow. - const failedDirs = glob.sync('*/failed/', { cwd: screenshotDir }); - for (const failedDir of failedDirs) { - cpSync(new URL(`./${failedDir}`, screenshotDir), new URL(`./${failedDir}`, artifactDir), { - force: true, - recursive: true, - }); - } - - const failedFiles = glob - .sync('*/failed/**/*.png', { cwd: artifactDir, ignore: '**/*-diff.png' }) - .map((p) => p.replace('/failed/', '/baseline/')); - for (const failedFile of failedFiles) { - const baselineFile = new URL(`./${failedFile}`, screenshotDir); - if (existsSync(baselineFile)) { - cpSync(baselineFile, new URL(`./${failedFile}`, artifactDir), { - force: true, - recursive: true, - }); - } - } -} else if ((platform() === 'linux' && !process.env.DEBUG) || process.env.FORCE_LOCAL) { - await startTestRunner(); -} else { - function executableIsAvailable(name: string): string | null { - try { - execSync(`${platform().startsWith('win') ? 'where' : 'which'} ${name}`, { encoding: 'utf8' }); - return name; - } catch (error) { - return null; - } - } - - const containerCmd = executableIsAvailable('podman') ?? executableIsAvailable('docker'); - if (!containerCmd) { - console.log('Either docker or podman need to be installed!'); - process.exit(1); - } - - const args = process.argv.slice(2); - const cwd = new URL('../../', import.meta.url); - const tag = 'lyne-vrt'; - const execOptions: ExecSyncOptionsWithStringEncoding = { - encoding: 'utf8', - stdio: 'inherit', - cwd, - }; - const branch = - process.env.GITHUB_REF_NAME ?? - process.env.BRANCH ?? - execSync('git rev-parse --abbrev-ref HEAD').toString().trim(); - execSync( - `${containerCmd} build ` + - '--file=tools/visual-regression-testing/testing.Dockerfile ' + - `--tag=${tag} .`, - execOptions, - ); - console.log(`\nTest image ready\n`); - mkdirSync(new URL('./dist/screenshots', cwd), { recursive: true }); - execSync( - `${containerCmd} run -it --rm --ipc=host ` + - `--env=BRANCH="${branch}"` + - `--volume=./dist/screenshots:/dist/screenshots ` + - `--entrypoint='["yarn", "wtr"${args.map((a) => `, "${a}"`).join('')}]' ` + - tag, - execOptions, - ); -} diff --git a/tools/visual-regression-testing/prepare-failed-screenshots-artifact.ts b/tools/visual-regression-testing/prepare-failed-screenshots-artifact.ts new file mode 100644 index 0000000000..0e4ae70fbd --- /dev/null +++ b/tools/visual-regression-testing/prepare-failed-screenshots-artifact.ts @@ -0,0 +1,30 @@ +import { cpSync, existsSync, mkdirSync, writeFileSync, type CopySyncOptions } from 'node:fs'; + +import * as glob from 'glob'; + +// When visual regression tests have failed, we only want to pack the relevant screenshots +// into the artifact transfered to the secure workflow, as uploading and downloading the full +// baseline would take far longer. +// Due to this we copy the necessary screenshots to /dist/screenshots-artifact which will +// be moved to /dist/screenshots in the secure workflow. + +const screenshotDir = new URL('../../dist/screenshots/', import.meta.url); +const artifactDir = new URL('../../dist/screenshots-artifact/', import.meta.url); +const copyOptions: CopySyncOptions = { force: true, recursive: true }; +mkdirSync(artifactDir, { recursive: true }); +writeFileSync(new URL('./.keep', artifactDir), '', 'utf8'); + +const failedDirs = glob.sync('*/failed/', { cwd: screenshotDir }); +for (const failedDir of failedDirs.map((d) => `./${d}`)) { + cpSync(new URL(failedDir, screenshotDir), new URL(failedDir, artifactDir), copyOptions); +} + +const failedFiles = glob + .sync('*/failed/**/*.png', { cwd: artifactDir, ignore: '**/*-diff.png' }) + .map((p) => p.replace('/failed/', '/baseline/')); +for (const failedFile of failedFiles.map((f) => `./${f}`)) { + const baselineFile = new URL(failedFile, screenshotDir); + if (existsSync(baselineFile)) { + cpSync(baselineFile, new URL(failedFile, artifactDir), copyOptions); + } +} diff --git a/tools/visual-regression-testing/testing.Dockerfile b/tools/visual-regression-testing/testing.Dockerfile deleted file mode 100644 index f56f1eb533..0000000000 --- a/tools/visual-regression-testing/testing.Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM mcr.microsoft.com/playwright:v1.44.0-jammy - -COPY package.json ./ -COPY yarn.lock ./ -RUN yarn install --frozen-lockfile --non-interactive - -COPY . . diff --git a/tools/visual-regression-testing/testing.Dockerfile.dockerignore b/tools/visual-regression-testing/testing.Dockerfile.dockerignore deleted file mode 100644 index 5de02b1469..0000000000 --- a/tools/visual-regression-testing/testing.Dockerfile.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -coverage -dist -node_modules \ No newline at end of file diff --git a/tools/web-test-runner/configure-container-playwright-browser.ts b/tools/web-test-runner/configure-container-playwright-browser.ts new file mode 100644 index 0000000000..9f3067de20 --- /dev/null +++ b/tools/web-test-runner/configure-container-playwright-browser.ts @@ -0,0 +1,56 @@ +import type { PlaywrightLauncher, ProductType } from '@web/test-runner-playwright'; +import type { Browser, LaunchOptions } from 'playwright'; +import * as playwright from 'playwright'; + +import { playwrightWebsocketAddress } from './container-playwright-browser-plugin.js'; + +interface PlaywrightLauncherPrivate { + // eslint-disable-next-line @typescript-eslint/naming-convention + __connectBrowserPromise: Promise | undefined; + browser: Browser | undefined; + product: ProductType; + launchOptions: LaunchOptions; +} + +/** + * Configure playwright browser execution to run the browsers in a container. + * + * @see https://github.com/microsoft/playwright/issues/26482 + * @param browser The playwright browser launcher. + */ +export function configureRemotePlaywrightBrowser(browser: PlaywrightLauncher): void { + // The original implementation calls launch instead of connect, + // so we overwrite the original method with this variant which + // calls connect with the websocket endpoint. + async function getOrStartBrowser(this: PlaywrightLauncherPrivate): Promise { + if (this.__connectBrowserPromise) { + return this.__connectBrowserPromise; + } + + if (!this.browser || !this.browser?.isConnected()) { + this.__connectBrowserPromise = (async () => { + // eslint-disable-next-line import-x/namespace + const browser = await playwright[this.product].connect(playwrightWebsocketAddress, { + headers: { 'x-playwright-launch-options': JSON.stringify(this.launchOptions) }, + }); + return browser; + })(); + const browser = await this.__connectBrowserPromise; + this.browser = browser; + this.__connectBrowserPromise = undefined; + } + return this.browser; + } + (browser as any).getOrStartBrowser = getOrStartBrowser; + + // Inside the container the dev server from `@web/test-runner` is not available + // so we need to adapt the address to point to the host machine. + const startSession = browser.startSession; + browser.startSession = async function (sessionId: string, url: string): Promise { + await startSession.call( + this, + sessionId, + url.replace('http://localhost:', 'http://host.containers.internal:'), + ); + }; +} diff --git a/tools/web-test-runner/container-playwright-browser-plugin.ts b/tools/web-test-runner/container-playwright-browser-plugin.ts new file mode 100644 index 0000000000..8693d4e5ab --- /dev/null +++ b/tools/web-test-runner/container-playwright-browser-plugin.ts @@ -0,0 +1,85 @@ +import { execSync, spawn } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { platform } from 'node:os'; + +import type { TestRunnerPlugin } from '@web/test-runner'; + +function executableIsAvailable(name: string): string | null { + try { + execSync(`${platform().startsWith('win') ? 'where' : 'which'} ${name}`, { encoding: 'utf8' }); + return name; + } catch (error) { + return null; + } +} + +const containerCmd = + process.env.CONTAINER_CMD ?? executableIsAvailable('podman') ?? executableIsAvailable('docker'); +const pkgJson = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')); +const playwrightVersion = pkgJson.devDependencies.playwright; +const port = 3000; +export const playwrightWebsocketAddress = `ws://localhost:${port}`; +// See https://github.com/microsoft/playwright/issues/26482 +export const startPlaywrightServerCommand = [ + containerCmd ?? 'docker', + 'run', + '-p', + `${port}:${port}`, + '--rm', + '--init', + '--workdir=/home/pwuser', + '--entrypoint=/bin/sh', + `mcr.microsoft.com/playwright:v${playwrightVersion}-jammy`, + `-c`, + `npx -y playwright@${playwrightVersion} run-server --port ${port} --host 0.0.0.0`, +]; + +// Reference: https://github.com/remcovaes/web-test-runner-vite-plugin +export function containerPlaywrightBrowserPlugin(): TestRunnerPlugin { + let abortController: AbortController; + + if (!containerCmd) { + console.log('Either docker or podman need to be installed!'); + process.exit(1); + } + + return { + name: 'remote-playwright-browser-plugin', + + async serverStart({ logger }) { + logger.log('Starting playwright browsers in a container'); + abortController = new AbortController(); + await new Promise((resolve, reject) => { + let id: NodeJS.Timeout | undefined = undefined; + spawn(startPlaywrightServerCommand[0], startPlaywrightServerCommand.slice(1), { + signal: abortController.signal, + }).on('error', (err) => { + clearInterval(id); + reject(err); + }); + id = setInterval(() => { + const timeout = AbortSignal.timeout(950); + const ws = new WebSocket(playwrightWebsocketAddress); + timeout.addEventListener('abort', () => ws.close()); + ws.addEventListener( + 'open', + () => { + console.log('Playwright container is ready'); + ws.close(); + clearInterval(id); + resolve(); + }, + { signal: timeout }, + ); + }, 1000); + AbortSignal.timeout(60000).addEventListener('abort', () => { + clearInterval(id); + reject('Failed to start container'); + }); + }); + }, + async serverStop() { + abortController.abort(); + }, + }; +} diff --git a/tools/web-test-runner/index.ts b/tools/web-test-runner/index.ts index b2cced322b..f15f5ed9bc 100644 --- a/tools/web-test-runner/index.ts +++ b/tools/web-test-runner/index.ts @@ -1,3 +1,5 @@ +export * from './configure-container-playwright-browser.js'; +export * from './container-playwright-browser-plugin.js'; export * from './minimal-reporter.js'; export * from './patched-summary-reporter.js'; export * from './visual-regression-plugin-config.js'; diff --git a/tools/web-test-runner/start-playwright-server.ts b/tools/web-test-runner/start-playwright-server.ts new file mode 100644 index 0000000000..fdda2fadbc --- /dev/null +++ b/tools/web-test-runner/start-playwright-server.ts @@ -0,0 +1,14 @@ +import { spawn } from 'node:child_process'; + +import { startPlaywrightServerCommand } from './container-playwright-browser-plugin.js'; + +const abortController = new AbortController(); +spawn(startPlaywrightServerCommand[0], startPlaywrightServerCommand.slice(1), { + stdio: 'inherit', + signal: abortController.signal, +}); + +process.on('SIGINT', () => { + abortController.abort(); + process.exit(0); +}); diff --git a/web-test-runner.config.ts b/web-test-runner.config.ts index 2e544abfc8..b0771355ab 100644 --- a/web-test-runner.config.ts +++ b/web-test-runner.config.ts @@ -1,3 +1,4 @@ +import { platform } from 'node:os'; import { parseArgs } from 'node:util'; import { litSsrPlugin } from '@lit-labs/testing/web-test-runner-ssr-plugin.js'; @@ -8,14 +9,20 @@ import { type TestRunnerGroupConfig, } from '@web/test-runner'; import { a11ySnapshotPlugin } from '@web/test-runner-commands/plugins'; -import { type PlaywrightLauncherArgs, playwrightLauncher } from '@web/test-runner-playwright'; +import { + type PlaywrightLauncherArgs, + playwrightLauncher, + type PlaywrightLauncher, +} from '@web/test-runner-playwright'; import { puppeteerLauncher } from '@web/test-runner-puppeteer'; import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin'; import { initCompiler } from 'sass'; import { + configureRemotePlaywrightBrowser, minimalReporter, patchedSummaryReporter, + containerPlaywrightBrowserPlugin, visualRegressionConfig, vitePlugin, } from './tools/web-test-runner/index.js'; @@ -23,6 +30,7 @@ import { const { values: cliArgs } = parseArgs({ strict: false, options: { + file: { type: 'string' }, ci: { type: 'boolean', default: !!process.env.CI }, debug: { type: 'boolean' }, 'all-browsers': { type: 'boolean', short: 'a' }, @@ -32,6 +40,8 @@ const { values: cliArgs } = parseArgs({ 'update-visual-baseline': { type: 'boolean' }, group: { type: 'string' }, ssr: { type: 'boolean' }, + container: { type: 'boolean' }, + local: { type: 'boolean' }, }, }); @@ -113,17 +123,34 @@ const suppressedLogs = [ '[vite] connected.', ]; +const testFile = typeof cliArgs.file === 'string' && cliArgs.file ? cliArgs.file : undefined; const groups: TestRunnerGroupConfig[] = [ - { name: 'ssr', files: 'src/**/*.ssr.spec.ts', testRunnerHtml }, + { name: 'ssr', files: testFile ?? 'src/**/*.ssr.spec.ts', testRunnerHtml }, ]; // The visual regression test group is only added when explicitly set, as the tests are very expensive. if (cliArgs.group === 'visual-regression') { - groups.push({ name: 'visual-regression', files: 'src/**/*.visual.spec.ts', testRunnerHtml }); + groups.push({ + name: 'visual-regression', + files: testFile ?? 'src/**/*.visual.spec.ts', + testRunnerHtml, + }); + if (!cliArgs.local && platform() !== 'linux') { + console.log( + `Running visual regression tests in a non-linux environment. Switching to container usage. Use --local to opt-out.`, + ); + cliArgs.container = true; + } +} + +if (cliArgs.container) { + browsers + .filter((b): b is PlaywrightLauncher => b.type === 'playwright') + .forEach((browser) => configureRemotePlaywrightBrowser(browser)); } export default { - files: ['src/**/*.spec.ts', '!**/*.{visual,ssr}.spec.ts'], + files: testFile ?? ['src/**/*.spec.ts', '!**/*.{visual,ssr}.spec.ts'], groups, nodeResolve: true, reporters: @@ -140,6 +167,7 @@ export default { ...visualRegressionConfig, update: !!cliArgs['update-visual-baseline'], }), + ...(cliArgs.container ? [containerPlaywrightBrowserPlugin()] : []), ], testFramework: { config: {