Merge pull request #19775 from storybookjs/chore/add-e2e-test-metadata

Maintenance: provide expected failure metadata in junit reports

.gitignore

@@ -7,6 +7,7 @@ dist
 *.DS_Store
 .cache
 junit.xml
+test-results
 /repros
 /sandbox
 .verdaccio-cache

code/playwright.config.ts

@@ -30,7 +30,17 @@ const config: PlaywrightTestConfig = {
   /* Opt out of parallel tests on CI. */
   workers: process.env.CI ? 1 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'html',
+  reporter: process.env.PLAYWRIGHT_JUNIT_OUTPUT_NAME
+    ? [
+        [
+          'junit',
+          {
+            embedAnnotationsAsProperties: true,
+            outputFile: process.env.PLAYWRIGHT_JUNIT_OUTPUT_NAME,
+          },
+        ],
+      ]
+    : 'html',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */

scripts/task.ts

@@ -1,13 +1,14 @@
 /* eslint-disable no-await-in-loop */
-import { AbortController } from 'node-abort-controller';
+import type { AbortController } from 'node-abort-controller';
 import { getJunitXml } from 'junit-xml';
-import { outputFile, existsSync, readFile } from 'fs-extra';
+import { outputFile, readFile, pathExists } from 'fs-extra';
 import { join, resolve } from 'path';
 import { prompt } from 'prompts';
 import boxen from 'boxen';
 import { dedent } from 'ts-dedent';
 
-import { createOptions, getCommand, getOptionsOrPrompt, OptionValues } from './utils/options';
+import type { OptionValues } from './utils/options';
+import { createOptions, getCommand, getOptionsOrPrompt } from './utils/options';
 import { install } from './tasks/install';
 import { compile } from './tasks/compile';
 import { check } from './tasks/check';
@@ -278,11 +279,15 @@ async function runTask(task: Task, details: TemplateDetails, optionValues: Passe
 
     return controller;
   } catch (err) {
-    if (junitFilename) await writeJunitXml(getTaskKey(task), details.key, startTime, err);
+    const hasJunitFile = await pathExists(junitFilename);
+    // If there's a non-test related error (junit report has not been reported already), we report the general failure in a junit report
+    if (junitFilename && !hasJunitFile) {
+      await writeJunitXml(getTaskKey(task), details.key, startTime, err);
+    }
 
     throw err;
   } finally {
-    if (existsSync(junitFilename)) {
+    if (await pathExists(junitFilename)) {
       const junitXml = await (await readFile(junitFilename)).toString();
       const prefixedXml = junitXml.replace(/classname="(.*)"/g, `classname="${details.key} $1"`);
       await outputFile(junitFilename, prefixedXml);

scripts/tasks/e2e-tests.ts

@@ -10,10 +10,8 @@ export const e2eTests: Task = {
     return false;
   },
   async run({ codeDir, junitFilename, template }, { dryRun, debug }) {
-    const reporter = process.env.CI ? 'junit' : 'html';
-
     await exec(
-      `yarn playwright test --reporter=${reporter}`,
+      `yarn playwright test`,
       {
         env: {
           STORYBOOK_URL: `http://localhost:${PORT}`,