diff --git a/.gitignore b/.gitignore index d5fedf28ff..034b8c2f30 100644 --- a/.gitignore +++ b/.gitignore @@ -61,4 +61,11 @@ package-lock.json # Examples /examples/carbon-for-ibm-products/*/node_modules -/examples/carbon-for-ibm-products/*/.yarn \ No newline at end of file +/examples/carbon-for-ibm-products/*/.yarn + +# Accessibility Verification Testing +.avt/ + +# Playwright +.playwright/ +/packages/ibm-products/.playwright/ \ No newline at end of file diff --git a/achecker.js b/achecker.js new file mode 100644 index 0000000000..1eee46a3b8 --- /dev/null +++ b/achecker.js @@ -0,0 +1,26 @@ +/** + * Copyright IBM Corp. 2024, 2024 + * + * This source code is licensed under the Apache-2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +'use strict'; + +const path = require('path'); + +module.exports = { + ruleArchive: 'latest', + policies: ['Custom_Ruleset'], + failLevels: ['violation'], + reportLevels: [ + 'violation', + 'potentialviolation', + 'recommendation', + 'potentialrecommendation', + 'manual', + ], + outputFormat: ['json'], + outputFolder: path.join('.avt', 'reports'), + baselineFolder: path.join('.avt', 'baseline'), +}; diff --git a/config/jest-config-ibm-cloud-cognitive/setup/matchers/toHaveNoACViolations.js b/config/jest-config-ibm-cloud-cognitive/setup/matchers/toHaveNoACViolations.js new file mode 100644 index 0000000000..f4f4ae97bf --- /dev/null +++ b/config/jest-config-ibm-cloud-cognitive/setup/matchers/toHaveNoACViolations.js @@ -0,0 +1,51 @@ +/** + * Copyright IBM Corp. 2024, 2024 + * + * This source code is licensed under the Apache-2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +'use strict'; + +let aChecker = null; + +async function toHaveNoACViolations(node, label) { + if (aChecker === null) { + aChecker = require('accessibility-checker'); + + const denylist = new Set([ + 'html_lang_exists', + 'page_title_exists', + 'skip_main_exists', + 'html_skipnav_exists', + 'aria_content_in_landmark', + 'aria_child_tabbable', + ]); + const ruleset = await aChecker.getRuleset('IBM_Accessibility'); + const customRuleset = JSON.parse(JSON.stringify(ruleset)); + + customRuleset.id = 'Custom_Ruleset'; + customRuleset.checkpoints = customRuleset.checkpoints.map((checkpoint) => { + checkpoint.rules = checkpoint.rules.filter((rule) => { + return !denylist.has(rule.id); + }); + return checkpoint; + }); + + aChecker.addRuleset(customRuleset); + } + + const results = await aChecker.getCompliance(node, label); + if (aChecker.assertCompliance(results.report) === 0) { + return { + pass: true, + }; + } else { + return { + pass: false, + message: () => aChecker.stringifyResults(results.report), + }; + } +} + +module.exports = toHaveNoACViolations; diff --git a/cspell.json b/cspell.json index 6e4409bf0b..82b7a43e29 100644 --- a/cspell.json +++ b/cspell.json @@ -117,12 +117,14 @@ "rowgroup", "rowheader", "rowindex", + "ruleset", "sbdocs", "scrollend", "semibold", "serializers", "setsize", "sidepanel", + "skipnav", "stackable", "stackblitz", "statusicon", diff --git a/docs/e2e_testing.md b/docs/e2e_testing.md new file mode 100644 index 0000000000..b4a2f8a9eb --- /dev/null +++ b/docs/e2e_testing.md @@ -0,0 +1,64 @@ +# e2e testing + +## Overview + +| Task | Command | +| :---------------------------------------------------- | :------------------------------------------------- | +| Run playwright tests | `yarn playwright test` | +| Run a specific playwright test | `yarn playwright test path/to/test-e2e.js` | +| Run playwright tests in a specific browser | `yarn playwright test --browser=chromium` | +| Run playwright tests in a specific project | `yarn playwright test --project=chromium` | +| Debug playwright tests | `yarn playwright test --debug` | +| Run playwright with browser visible | `yarn playwright test --project=chromium --headed` | +| Run playwright tests that match a specific tag | `yarn playwright test --grep @tag-name` | +| Run playwright tests that do not match a specific tag | `yarn playwright test --grep-invert @tag-name` | + +### Playwright + +We use Playwright to run end-to-end tests against components in `ibm-products`, +together with the `accessibility-checker` package from IBM Accessibility to +ensure we catch and prevent a11y violations in the components we ship. + +These tests are authored within the `e2e` directory and match the file pattern: +`*.test.avt.e2e.js`. + +### Tags + +Playwright tests are divided into different tag categories for reporting +purposes. We currently only support `@avt` tags but in future release will be +exploring a `@vrt` tag to denote visual regression testing. + +For avt tests, the test title should always include one of the following: + +| Tag | Description | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `@avt` | High level/root tag that should wrap all avt tests. This is usually placed in a `describe` block title. | +| `@avt-default-state` | Sub-tag of `@avt`, used to tag individual tests covering the default state of a component. | +| `@avt-advanced-states` | Sub-tag of `@avt`, used to tag individual tests covering advanced states of a component (open/close, invalid, expanded, etc.). | +| `@avt-keyboard-nav` | Sub-tag of `@avt`, used to tag individual tests covering keyboard navigation flows. | + +#### Developing + +When working with Playwright locally, it's important to start up the service +that you're testing against. For components in `@carbon/ibm-products`, this will +mean starting up the storybook locally by doing the following from the root of +the project: + +```bash +yarn +yarn storybook +``` + +If this is the first time you're running playwright tests in the project, you +will also need to run the follow installation command so that Playwright has the +browsers specified in `playwright.config.js` to run the e2e tests: + +```bash +yarn playwright install +``` + +Now you can run the playwright tests with one of the commands in the overview +table above. This is an ongoing effort our team will be working on with some +additions including adding these e2e tests as part of our PR checks, including +additional `@avt` coverage for other components, and introducing visual +regression testing by using playwright and Percy together. diff --git a/e2e/components/Cascade/Cascade.test.avt.e2e.js b/e2e/components/Cascade/Cascade.test.avt.e2e.js new file mode 100644 index 0000000000..ba67d50f6d --- /dev/null +++ b/e2e/components/Cascade/Cascade.test.avt.e2e.js @@ -0,0 +1,24 @@ +/** + * Copyright IBM Corp. 2024, 2024 + * + * This source code is licensed under the Apache-2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +'use strict'; + +import { expect, test } from '@playwright/test'; +import { visitStory } from '../../test-utils/storybook'; + +test.describe('Cascade @avt', () => { + test('@avt-default-state', async ({ page }) => { + await visitStory(page, { + component: 'Cascade', + id: 'ibm-products-patterns-cascade-cascade--without-grid', + globals: { + carbonTheme: 'white', + }, + }); + await expect(page).toHaveNoACViolations('Cascade @avt-default-state'); + }); +}); diff --git a/e2e/test-utils/storybook.js b/e2e/test-utils/storybook.js new file mode 100644 index 0000000000..15f3ca1fd5 --- /dev/null +++ b/e2e/test-utils/storybook.js @@ -0,0 +1,50 @@ +/** + * Copyright IBM Corp. 2024, 2024 + * + * This source code is licensed under the Apache-2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +async function visitStory(page, options) { + const { component, story, id, globals, args } = options; + let url = getStoryUrl({ + component, + story, + id, + }); + + if (args) { + const values = Object.entries(args) + .map(([key, value]) => { + return `${key}:${value}`; + }) + .join(';'); + url = url + `&args=${values}`; + } + + if (globals) { + const values = Object.entries(globals) + .map(([key, value]) => { + return `${key}:${value}`; + }) + .join(','); + url = url + `&globals=${values}`; + } + + await page.goto(url); +} + +function getStoryUrl({ component, story, id }) { + const normalized = id ? id : `ibm-products-components-${component}--${story}`; // TODO: refactor this because we have some story id's prefixed with `ibm-products-patterns-${component}--${story}` + + // Note: We serve a static storybook in CI that will trim .html extensions + // from the URL + if (process.env.CI) { + return `/iframe?id=${normalized}&viewMode=story`; + } + return `/iframe.html?id=${normalized}&viewMode=story`; +} + +module.exports = { + visitStory, +}; diff --git a/package.json b/package.json index a0d5efc047..770d251616 100644 --- a/package.json +++ b/package.json @@ -63,12 +63,14 @@ "@babel/preset-react": "^7.17.12", "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", + "@playwright/test": "^1.36.2", "@testing-library/dom": "^8.11.4", "@testing-library/react": "^14.0.0", "@testing-library/react-hooks": "^8.0.1", "@testing-library/user-event": "^14.4.3", "@typescript-eslint/eslint-plugin": "^6.21.0", "@typescript-eslint/parser": "^6.21.0", + "accessibility-checker": "^3.1.65", "cheerio": "^1.0.0-rc.12", "copyfiles": "^2.4.1", "cspell": "^8.3.2", diff --git a/playwright.config.js b/playwright.config.js new file mode 100644 index 0000000000..b4702d4cad --- /dev/null +++ b/playwright.config.js @@ -0,0 +1,115 @@ +/** + * Copyright IBM Corp. 2024, 2024 + * + * This source code is licensed under the Apache-2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +const { devices, expect } = require('@playwright/test'); +const path = require('path'); + +const config = { + // https://playwright.dev/docs/api/class-testconfig#test-config-test-dir + testDir: path.join(__dirname, 'e2e'), + + // https://playwright.dev/docs/api/class-testconfig#test-config-test-ignore + testIgnore: [], + + // https://playwright.dev/docs/api/class-testconfig#test-config-test-match + testMatch: /.*.test(.avt|.vrt)?.e2e\.m?js$/, + + // https://playwright.dev/docs/api/class-testconfig#test-config-timeout + timeout: 10000 * 30, + + // https://playwright.dev/docs/test-timeouts + expect: { timeout: 100000 }, + + // https://playwright.dev/docs/api/class-testconfig#test-config-output-dir + outputDir: path.join(__dirname, '.playwright', 'results'), + snapshotDir: path.join(__dirname, '.playwright', 'snapshots'), + + // https://playwright.dev/docs/test-parallel#parallelize-tests-in-a-single-file + // fullyParallel: true, + + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + use: { + baseURL: 'http://localhost:3000', + trace: 'on-first-retry', + }, + projects: [ + // Desktop + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'], + }, + }, + ], + reporter: [ + ['line'], + [ + 'json', + { + outputFile: path.join(__dirname, '.playwright', 'results.json'), + }, + ], + [ + 'json', + { + outputFile: path.join( + __dirname, + 'packages/ibm-products/.playwright', + 'INTERNAL_AVT_REPORT_DO_NOT_USE.json' + ), + }, + ], + ], +}; + +let aChecker; + +expect.extend({ + async toHaveNoACViolations(page, id) { + if (!aChecker) { + aChecker = require('accessibility-checker'); + const denylist = new Set([ + 'html_lang_exists', + 'page_title_exists', + 'skip_main_exists', + 'html_skipnav_exists', + 'aria_content_in_landmark', + 'aria_child_tabbable', + 'skip_main_described' + ]); + + const ruleset = await aChecker.getRuleset('IBM_Accessibility'); + const customRuleset = JSON.parse(JSON.stringify(ruleset)); + + customRuleset.id = 'Custom_Ruleset'; + customRuleset.checkpoints = customRuleset.checkpoints.map( + (checkpoint) => { + checkpoint.rules = checkpoint.rules.filter((rule) => { + return !denylist.has(rule.id); + }); + return checkpoint; + }); + + aChecker.addRuleset(customRuleset); + } + + const result = await aChecker.getCompliance(page, id); + if (aChecker.assertCompliance(result.report) === 0) { + return { + pass: true, + }; + } else { + return { + pass: false, + message: () => aChecker.stringifyResults(result.report), + }; + } + }, +}); + +module.exports = config; diff --git a/yarn.lock b/yarn.lock index 0936bff1ff..066755e4bf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4425,6 +4425,17 @@ __metadata: languageName: node linkType: hard +"@playwright/test@npm:^1.36.2": + version: 1.41.2 + resolution: "@playwright/test@npm:1.41.2" + dependencies: + playwright: "npm:1.41.2" + bin: + playwright: cli.js + checksum: e87405987fa024f75acc223c47fcb2da0a66b2fa0cd9a583ca5b02aac12be353d0c262bf6a22b9bc40550c86c8b7629e70cd27f508ec370d9c92bb72f74581e7 + languageName: node + linkType: hard + "@pmmmwh/react-refresh-webpack-plugin@npm:^0.5.11": version: 0.5.11 resolution: "@pmmmwh/react-refresh-webpack-plugin@npm:0.5.11" @@ -7540,8 +7551,8 @@ __metadata: linkType: hard "accessibility-checker@npm:^3.1.65": - version: 3.1.65 - resolution: "accessibility-checker@npm:3.1.65" + version: 3.1.67 + resolution: "accessibility-checker@npm:3.1.67" dependencies: axios: "npm:^1.4.0" chromedriver: "npm:*" @@ -7552,7 +7563,7 @@ __metadata: string-hash: "npm:^1.1.3" bin: achecker: bin/achecker.js - checksum: c11f06802995b620e8ca08eb809f053f88a167fce5abc07f75258415824edb48ffac18acb89e19451dec8d4efcdf2384592a0f91576e0178bddbd7c338c8ac8f + checksum: d8768bfc595d282cace6f19d2e15a10b8113284e24640448caa716c4ea91c7d3f11aa5c861c82f91d4be31e37577974be11597429f5fe2402546160761ac3451 languageName: node linkType: hard @@ -12866,6 +12877,16 @@ __metadata: languageName: node linkType: hard +"fsevents@npm:2.3.2": + version: 2.3.2 + resolution: "fsevents@npm:2.3.2" + dependencies: + node-gyp: "npm:latest" + checksum: 6b5b6f5692372446ff81cf9501c76e3e0459a4852b3b5f1fc72c103198c125a6b8c72f5f166bdd76ffb2fca261e7f6ee5565daf80dca6e571e55bcc589cc1256 + conditions: os=darwin + languageName: node + linkType: hard + "fsevents@npm:^2.3.2, fsevents@npm:~2.3.2": version: 2.3.3 resolution: "fsevents@npm:2.3.3" @@ -12876,6 +12897,15 @@ __metadata: languageName: node linkType: hard +"fsevents@patch:fsevents@npm%3A2.3.2#optional!builtin": + version: 2.3.2 + resolution: "fsevents@patch:fsevents@npm%3A2.3.2#optional!builtin::version=2.3.2&hash=df0bf1" + dependencies: + node-gyp: "npm:latest" + conditions: os=darwin + languageName: node + linkType: hard + "fsevents@patch:fsevents@npm%3A^2.3.2#optional!builtin, fsevents@patch:fsevents@npm%3A~2.3.2#optional!builtin": version: 2.3.3 resolution: "fsevents@patch:fsevents@npm%3A2.3.3#optional!builtin::version=2.3.3&hash=df0bf1" @@ -13956,12 +13986,14 @@ __metadata: "@babel/preset-react": "npm:^7.17.12" "@commitlint/cli": "npm:^18.6.0" "@commitlint/config-conventional": "npm:^18.6.0" + "@playwright/test": "npm:^1.36.2" "@testing-library/dom": "npm:^8.11.4" "@testing-library/react": "npm:^14.0.0" "@testing-library/react-hooks": "npm:^8.0.1" "@testing-library/user-event": "npm:^14.4.3" "@typescript-eslint/eslint-plugin": "npm:^6.21.0" "@typescript-eslint/parser": "npm:^6.21.0" + accessibility-checker: "npm:^3.1.65" cheerio: "npm:^1.0.0-rc.12" copyfiles: "npm:^2.4.1" cspell: "npm:^8.3.2" @@ -19896,6 +19928,30 @@ __metadata: languageName: node linkType: hard +"playwright-core@npm:1.41.2": + version: 1.41.2 + resolution: "playwright-core@npm:1.41.2" + bin: + playwright-core: cli.js + checksum: 77ff881ebb9cc0654edd00c5ff202f5f61aee7a5318e1f12a82e30a3636de21e8b5982fae6138e5bb90115ae509c15a640cf85b10b3e2daebb2bb286da54fd4c + languageName: node + linkType: hard + +"playwright@npm:1.41.2": + version: 1.41.2 + resolution: "playwright@npm:1.41.2" + dependencies: + fsevents: "npm:2.3.2" + playwright-core: "npm:1.41.2" + dependenciesMeta: + fsevents: + optional: true + bin: + playwright: cli.js + checksum: 272399f622dc2df90fbef147b9b1cfab5d7a78cc364bdfa98d2bf08faa9894346f58629fe4fef41b108ca2cb203b3970d7886b7f392cb0399c75b521478e2920 + languageName: node + linkType: hard + "polished@npm:^4.2.2": version: 4.2.2 resolution: "polished@npm:4.2.2"