test(e2e): add e2e testing setup with playwright (#4345)

* test: add e2e testing setup with playwright

* chore: update copyright header

* docs: add initial e2e docs

* docs: add installation step doc

.gitignore

@@ -61,4 +61,11 @@ package-lock.json
 
 # Examples
 /examples/carbon-for-ibm-products/*/node_modules
-/examples/carbon-for-ibm-products/*/.yarn
+/examples/carbon-for-ibm-products/*/.yarn
+
+# Accessibility Verification Testing
+.avt/
+
+# Playwright
+.playwright/
+/packages/ibm-products/.playwright/

achecker.js

@@ -0,0 +1,26 @@
+/**
+ * Copyright IBM Corp. 2024, 2024
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+const path = require('path');
+
+module.exports = {
+  ruleArchive: 'latest',
+  policies: ['Custom_Ruleset'],
+  failLevels: ['violation'],
+  reportLevels: [
+    'violation',
+    'potentialviolation',
+    'recommendation',
+    'potentialrecommendation',
+    'manual',
+  ],
+  outputFormat: ['json'],
+  outputFolder: path.join('.avt', 'reports'),
+  baselineFolder: path.join('.avt', 'baseline'),
+};

config/jest-config-ibm-cloud-cognitive/setup/matchers/toHaveNoACViolations.js

@@ -0,0 +1,51 @@
+/**
+ * Copyright IBM Corp. 2024, 2024
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+let aChecker = null;
+
+async function toHaveNoACViolations(node, label) {
+  if (aChecker === null) {
+    aChecker = require('accessibility-checker');
+
+    const denylist = new Set([
+      'html_lang_exists',
+      'page_title_exists',
+      'skip_main_exists',
+      'html_skipnav_exists',
+      'aria_content_in_landmark',
+      'aria_child_tabbable',
+    ]);
+    const ruleset = await aChecker.getRuleset('IBM_Accessibility');
+    const customRuleset = JSON.parse(JSON.stringify(ruleset));
+
+    customRuleset.id = 'Custom_Ruleset';
+    customRuleset.checkpoints = customRuleset.checkpoints.map((checkpoint) => {
+      checkpoint.rules = checkpoint.rules.filter((rule) => {
+        return !denylist.has(rule.id);
+      });
+      return checkpoint;
+    });
+
+    aChecker.addRuleset(customRuleset);
+  }
+
+  const results = await aChecker.getCompliance(node, label);
+  if (aChecker.assertCompliance(results.report) === 0) {
+    return {
+      pass: true,
+    };
+  } else {
+    return {
+      pass: false,
+      message: () => aChecker.stringifyResults(results.report),
+    };
+  }
+}
+
+module.exports = toHaveNoACViolations;

cspell.json

@@ -117,12 +117,14 @@
     "rowgroup",
     "rowheader",
     "rowindex",
+    "ruleset",
     "sbdocs",
     "scrollend",
     "semibold",
     "serializers",
     "setsize",
     "sidepanel",
+    "skipnav",
     "stackable",
     "stackblitz",
     "statusicon",

docs/e2e_testing.md

@@ -0,0 +1,64 @@
+# e2e testing
+
+## Overview
+
+| Task                                                  | Command                                            |
+| :---------------------------------------------------- | :------------------------------------------------- |
+| Run playwright tests                                  | `yarn playwright test`                             |
+| Run a specific playwright test                        | `yarn playwright test path/to/test-e2e.js`         |
+| Run playwright tests in a specific browser            | `yarn playwright test --browser=chromium`          |
+| Run playwright tests in a specific project            | `yarn playwright test --project=chromium`          |
+| Debug playwright tests                                | `yarn playwright test --debug`                     |
+| Run playwright with browser visible                   | `yarn playwright test --project=chromium --headed` |
+| Run playwright tests that match a specific tag        | `yarn playwright test --grep @tag-name`            |
+| Run playwright tests that do not match a specific tag | `yarn playwright test --grep-invert @tag-name`     |
+
+### Playwright
+
+We use Playwright to run end-to-end tests against components in `ibm-products`,
+together with the `accessibility-checker` package from IBM Accessibility to
+ensure we catch and prevent a11y violations in the components we ship.
+
+These tests are authored within the `e2e` directory and match the file pattern:
+`*.test.avt.e2e.js`.
+
+### Tags
+
+Playwright tests are divided into different tag categories for reporting
+purposes. We currently only support `@avt` tags but in future release will be
+exploring a `@vrt` tag to denote visual regression testing.
+
+For avt tests, the test title should always include one of the following:
+
+| Tag                    | Description                                                                                                                    |
+| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
+| `@avt`                 | High level/root tag that should wrap all avt tests. This is usually placed in a `describe` block title.                        |
+| `@avt-default-state`   | Sub-tag of `@avt`, used to tag individual tests covering the default state of a component.                                     |
+| `@avt-advanced-states` | Sub-tag of `@avt`, used to tag individual tests covering advanced states of a component (open/close, invalid, expanded, etc.). |
+| `@avt-keyboard-nav`    | Sub-tag of `@avt`, used to tag individual tests covering keyboard navigation flows.                                            |
+
+#### Developing
+
+When working with Playwright locally, it's important to start up the service
+that you're testing against. For components in `@carbon/ibm-products`, this will
+mean starting up the storybook locally by doing the following from the root of
+the project:
+
+```bash
+yarn
+yarn storybook
+```
+
+If this is the first time you're running playwright tests in the project, you
+will also need to run the follow installation command so that Playwright has the
+browsers specified in `playwright.config.js` to run the e2e tests:
+
+```bash
+yarn playwright install
+```
+
+Now you can run the playwright tests with one of the commands in the overview
+table above. This is an ongoing effort our team will be working on with some
+additions including adding these e2e tests as part of our PR checks, including
+additional `@avt` coverage for other components, and introducing visual
+regression testing by using playwright and Percy together.

e2e/components/Cascade/Cascade.test.avt.e2e.js

@@ -0,0 +1,24 @@
+/**
+ * Copyright IBM Corp. 2024, 2024
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+import { expect, test } from '@playwright/test';
+import { visitStory } from '../../test-utils/storybook';
+
+test.describe('Cascade @avt', () => {
+  test('@avt-default-state', async ({ page }) => {
+    await visitStory(page, {
+      component: 'Cascade',
+      id: 'ibm-products-patterns-cascade-cascade--without-grid',
+      globals: {
+        carbonTheme: 'white',
+      },
+    });
+    await expect(page).toHaveNoACViolations('Cascade @avt-default-state');
+  });
+});

e2e/test-utils/storybook.js

@@ -0,0 +1,50 @@
+/**
+ * Copyright IBM Corp. 2024, 2024
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+async function visitStory(page, options) {
+  const { component, story, id, globals, args } = options;
+  let url = getStoryUrl({
+    component,
+    story,
+    id,
+  });
+
+  if (args) {
+    const values = Object.entries(args)
+      .map(([key, value]) => {
+        return `${key}:${value}`;
+      })
+      .join(';');
+    url = url + `&args=${values}`;
+  }
+
+  if (globals) {
+    const values = Object.entries(globals)
+      .map(([key, value]) => {
+        return `${key}:${value}`;
+      })
+      .join(',');
+    url = url + `&globals=${values}`;
+  }
+
+  await page.goto(url);
+}
+
+function getStoryUrl({ component, story, id }) {
+  const normalized = id ? id : `ibm-products-components-${component}--${story}`; // TODO: refactor this because we have some story id's prefixed with `ibm-products-patterns-${component}--${story}`
+
+  // Note: We serve a static storybook in CI that will trim .html extensions
+  // from the URL
+  if (process.env.CI) {
+    return `/iframe?id=${normalized}&viewMode=story`;
+  }
+  return `/iframe.html?id=${normalized}&viewMode=story`;
+}
+
+module.exports = {
+  visitStory,
+};

package.json

@@ -63,12 +63,14 @@
     "@babel/preset-react": "^7.17.12",
     "@commitlint/cli": "^18.6.0",
     "@commitlint/config-conventional": "^18.6.0",
+    "@playwright/test": "^1.36.2",
     "@testing-library/dom": "^8.11.4",
     "@testing-library/react": "^14.0.0",
     "@testing-library/react-hooks": "^8.0.1",
     "@testing-library/user-event": "^14.4.3",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^6.21.0",
+    "accessibility-checker": "^3.1.65",
     "cheerio": "^1.0.0-rc.12",
     "copyfiles": "^2.4.1",
     "cspell": "^8.3.2",

playwright.config.js

@@ -0,0 +1,115 @@
+/**
+ * Copyright IBM Corp. 2024, 2024
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const { devices, expect } = require('@playwright/test');
+const path = require('path');
+
+const config = {
+  // https://playwright.dev/docs/api/class-testconfig#test-config-test-dir
+  testDir: path.join(__dirname, 'e2e'),
+
+  // https://playwright.dev/docs/api/class-testconfig#test-config-test-ignore
+  testIgnore: [],
+
+  // https://playwright.dev/docs/api/class-testconfig#test-config-test-match
+  testMatch: /.*.test(.avt|.vrt)?.e2e\.m?js$/,
+
+  // https://playwright.dev/docs/api/class-testconfig#test-config-timeout
+  timeout: 10000 * 30,
+
+  // https://playwright.dev/docs/test-timeouts
+  expect: { timeout: 100000 },
+
+  // https://playwright.dev/docs/api/class-testconfig#test-config-output-dir
+  outputDir: path.join(__dirname, '.playwright', 'results'),
+  snapshotDir: path.join(__dirname, '.playwright', 'snapshots'),
+
+  // https://playwright.dev/docs/test-parallel#parallelize-tests-in-a-single-file
+  // fullyParallel: true,
+
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  use: {
+    baseURL: 'http://localhost:3000',
+    trace: 'on-first-retry',
+  },
+  projects: [
+    // Desktop
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+  ],
+  reporter: [
+    ['line'],
+    [
+      'json',
+      {
+        outputFile: path.join(__dirname, '.playwright', 'results.json'),
+      },
+    ],
+    [
+      'json',
+      {
+        outputFile: path.join(
+          __dirname,
+          'packages/ibm-products/.playwright',
+          'INTERNAL_AVT_REPORT_DO_NOT_USE.json'
+        ),
+      },
+    ],
+  ],
+};
+
+let aChecker;
+
+expect.extend({
+  async toHaveNoACViolations(page, id) {
+    if (!aChecker) {
+      aChecker = require('accessibility-checker');
+      const denylist = new Set([
+        'html_lang_exists',
+        'page_title_exists',
+        'skip_main_exists',
+        'html_skipnav_exists',
+        'aria_content_in_landmark',
+        'aria_child_tabbable',
+        'skip_main_described'
+      ]);
+
+      const ruleset = await aChecker.getRuleset('IBM_Accessibility');
+      const customRuleset = JSON.parse(JSON.stringify(ruleset));
+
+      customRuleset.id = 'Custom_Ruleset';
+      customRuleset.checkpoints = customRuleset.checkpoints.map(
+        (checkpoint) => {
+          checkpoint.rules = checkpoint.rules.filter((rule) => {
+            return !denylist.has(rule.id);
+          });
+          return checkpoint;
+        });
+
+      aChecker.addRuleset(customRuleset);
+    }
+
+    const result = await aChecker.getCompliance(page, id);
+    if (aChecker.assertCompliance(result.report) === 0) {
+      return {
+        pass: true,
+      };
+    } else {
+      return {
+        pass: false,
+        message: () => aChecker.stringifyResults(result.report),
+      };
+    }
+  },
+});
+
+module.exports = config;