diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d873ddc0909b..ab973e199d2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,7 +165,7 @@ jobs: if: always() uses: actions/upload-artifact@v3 with: - name: playwright-report + name: playwright-vrt-report path: .playwright vrt: @@ -221,5 +221,5 @@ jobs: if: always() uses: actions/upload-artifact@v3 with: - name: playwright-report + name: playwright-avt-report path: .playwright diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 629bd59bae77..05d1a7db0207 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,9 +29,40 @@ jobs: - name: Install dependencies run: yarn install --immutable --immutable-cache --check-cache + # For generating playwright json report for avt tests + - name: Install browsers + run: yarn playwright install --with-deps + - name: Build project run: yarn build + # Additional steps for generating playwright json report for avt tests + - name: Build storybook + run: + STORYBOOK_STORE_7=false yarn workspace @carbon/react storybook:build + - name: Run storybook + id: storybook + run: | + npx serve -l 3000 packages/react/storybook-static & + pid=$! + echo ::set-output name=pid::"$pid" + - uses: ./actions/wait-for-it + with: + URL: 'http://localhost:3000' + timeout-minutes: 3 + - name: Run AVT + if: github.repository == 'carbon-design-system/carbon' + run: | + yarn playwright test --project chromium --grep @avt + - name: Stop storybook + run: kill ${{ steps.storybook.outputs.pid }} + - name: Upload test results + if: always() + uses: actions/upload-artifact@v3 + with: + name: playwright-avt-report + path: .playwright + - name: Run Continuous Integration checks run: yarn ci-check diff --git a/docs/testing.md b/docs/testing.md index d985ca225bc1..9f2237de7ee0 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -37,6 +37,21 @@ in our elements site. These tests are authored within the `e2e` directory and match the file pattern: `*-test.e2e.js`. +#### Tags + +Playwright tests are divided into different tag categories for reporting +purposes. `@avt` test tags are used to populate accessibility test statuses +within component pages on https://carbondesignsystem.com + +For avt tests, the test title should always include one of the following: + +| Tag | Description | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `@avt` | High level/root tag that should wrap all avt tests. This is usually placed in a `describe` block title. | +| `@avt-default-state` | Sub-tag of `@avt`, used to tag individual tests covering the default state of a component. | +| `@avt-advanced-states` | Sub-tag of `@avt`, used to tag individual tests covering advanced states of a component (open/close, invalid, expanded, etc.). | +| `@avt-keyboard-nav` | Sub-tag of `@avt`, used to tag individual tests covering keyboard navigation flows. | + #### Developing When working with Playwright locally, it's important to start up the service diff --git a/e2e/components/accordion/accordion-test.avt.e2e.js b/e2e/components/accordion/accordion-test.avt.e2e.js index d22e7b53d2f8..44fa7dd98755 100644 --- a/e2e/components/accordion/accordion-test.avt.e2e.js +++ b/e2e/components/accordion/accordion-test.avt.e2e.js @@ -11,7 +11,7 @@ import { expect, test } from '@playwright/test'; import { visitStory } from '../../test-utils/storybook'; test.describe('Accordion @avt', () => { - test('accessibility-checker default', async ({ page }) => { + test('@avt-default-state', async ({ page }) => { await visitStory(page, { component: 'Accordion', id: 'components-accordion--default', @@ -19,10 +19,10 @@ test.describe('Accordion @avt', () => { theme: 'white', }, }); - await expect(page).toHaveNoACViolations('accordion'); + await expect(page).toHaveNoACViolations('Accordion @avt-default-state'); }); - test('accessibility-checker skeleton', async ({ page }) => { + test('@avt-advanced-states skeleton', async ({ page }) => { await visitStory(page, { component: 'Accordion', id: 'components-accordion--skeleton', @@ -30,10 +30,12 @@ test.describe('Accordion @avt', () => { theme: 'white', }, }); - await expect(page).toHaveNoACViolations('accordion--skeleton'); + await expect(page).toHaveNoACViolations( + 'Accordion @avt-advanced-states skeleton' + ); }); - test('accessibility-checker with layer', async ({ page }) => { + test('@avt-advanced-states with layer', async ({ page }) => { await visitStory(page, { component: 'Accordion', id: 'components-accordion--with-layer', @@ -41,10 +43,12 @@ test.describe('Accordion @avt', () => { theme: 'white', }, }); - await expect(page).toHaveNoACViolations('accordion--with-layer'); + await expect(page).toHaveNoACViolations( + 'Accordion @avt-advanced-states with layer' + ); }); - test('accessibility-checker keyboard nav', async ({ page }) => { + test('@avt-keyboard-nav', async ({ page }) => { await visitStory(page, { component: 'Accordion', id: 'components-accordion--default', @@ -78,7 +82,7 @@ test.describe('Accordion @avt', () => { await expect(accordin_btn2).toHaveAttribute('aria-expanded', 'false'); }); - test('accessibility-checker mouse click', async ({ page }) => { + test('@avt-advanced-states mouse click', async ({ page }) => { await visitStory(page, { component: 'Accordion', id: 'components-accordion--default', @@ -91,12 +95,14 @@ test.describe('Accordion @avt', () => { await expect(accordin_btn1).toHaveAttribute('aria-expanded', 'true'); // Checking for ACViolation - await expect(page).toHaveNoACViolations('accordion-mouse-click'); + await expect(page).toHaveNoACViolations( + 'Accordion @avt-advanced-states mouse click' + ); await accordin_btn1.click(); await expect(accordin_btn1).toHaveAttribute('aria-expanded', 'false'); }); - test('accessibility-checker mouse hover', async ({ page }) => { + test('@avt-advanced-states mouse hover', async ({ page }) => { await visitStory(page, { component: 'Accordion', id: 'components-accordion--default', @@ -106,10 +112,12 @@ test.describe('Accordion @avt', () => { }); const accordin_btn1 = page.getByRole('button', { name: 'Section 1 title' }); await accordin_btn1.hover(); - await expect(page).toHaveNoACViolations('accordion-mouse-hover'); + await expect(page).toHaveNoACViolations( + 'Accordion @avt-advanced-states mouse hover' + ); }); - test('accessibility-checker disabled accordion', async ({ page }) => { + test('@avt-advanced-states disabled', async ({ page }) => { await visitStory(page, { component: 'Accordion', id: 'components-accordion--playground', @@ -122,6 +130,8 @@ test.describe('Accordion @avt', () => { }); const accordin_btn1 = page.getByRole('button', { name: 'Section 1 title' }); await expect(accordin_btn1).toBeDisabled(); - await expect(page).toHaveNoACViolations('accordion-disabled'); + await expect(page).toHaveNoACViolations( + 'Accordion @avt-advanced-states disabled' + ); }); }); diff --git a/packages/react/package.json b/packages/react/package.json index 6aee45f0f807..8dbcf892db13 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -18,7 +18,8 @@ "index.scss", "icons/index.js", "icons/index.esm.js", - "icons/package.json" + "icons/package.json", + ".playwright/INTERNAL_AVT_REPORT_DO_NOT_USE.json" ], "keywords": [ "ibm", diff --git a/playwright.config.js b/playwright.config.js index 947cfaede49d..634593a73047 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -54,6 +54,16 @@ const config = { outputFile: path.join(__dirname, '.playwright', 'results.json'), }, ], + [ + 'json', + { + outputFile: path.join( + __dirname, + 'packages/react/.playwright', + 'INTERNAL_AVT_REPORT_DO_NOT_USE.json' + ), + }, + ], ], };