test(e2e): add proper tags to avt tests (#14915)

* test(e2e): add proper tags to avt tests * ci(avt): shard avt tests * fix: typo * test(config): increase playwright expect timeout * test(playwright): increase timeout config * test(progressindicator): update to use proper tags * test(progressindicator): correct focus order --------- Co-authored-by: Andrea N. Cardona <[email protected]> Co-authored-by: Alison Joseph <[email protected]> Co-authored-by: TJ Egan <[email protected]>
carbon-design-system · Nov 8, 2023 · 5f4f2ad · 5f4f2ad
1 parent 5b6ba55
commit 5f4f2ad
Show file tree

Hide file tree

Showing 75 changed files with 773 additions and 724 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -180,7 +180,10 @@ jobs:
         if: ${{ needs.vrt-runner.result != 'success' }}
         run: exit 1
 
-  avt:
+  avt-runner:
+    strategy:
+      matrix:
+        shard: [1, 2, 3, 4]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
@@ -210,15 +213,15 @@ jobs:
         run: |
           npx serve -l 3000 packages/react/storybook-static &
           pid=$!
-          echo ::set-output name=pid::"$pid"
+          echo "pid=$pid" >> $GITHUB_OUTPUT
       - uses: ./actions/wait-for-it
         with:
           URL: 'http://localhost:3000'
         timeout-minutes: 3
       - name: Run AVT
         if: github.repository == 'carbon-design-system/carbon'
         run: |
-          yarn playwright test --project chromium --grep @avt
+          yarn playwright test --project chromium --grep @avt --shard="${{ matrix.shard }}/4"
       - name: Stop storybook
         run: kill ${{ steps.storybook.outputs.pid }}
       - name: Upload test results
@@ -227,3 +230,12 @@ jobs:
         with:
           name: playwright-avt-report
           path: .playwright
+
+  avt:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: avt-runner
+    steps:
+      - name: Check AVT Runner job status
+        if: ${{ needs.avt-runner.result != 'success' }}
+        run: exit 1
diff --git a/README.md b/README.md
diff --git a/...nents/accordion/accordion-test.avt.e2e.js → ...nents/Accordion/Accordion-test.avt.e2e.js b/...nents/accordion/accordion-test.avt.e2e.js → ...nents/Accordion/Accordion-test.avt.e2e.js
diff --git a/...omponents/accordion/accordion-test.e2e.js → ...omponents/Accordion/Accordion-test.e2e.js b/...omponents/accordion/accordion-test.e2e.js → ...omponents/Accordion/Accordion-test.e2e.js
diff --git a/e2e/components/AspectRatio/AspectRatio-test.avt.e2e.js b/e2e/components/AspectRatio/AspectRatio-test.avt.e2e.js
@@ -0,0 +1,24 @@
+/**
+ * Copyright IBM Corp. 2016, 2023
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+const { expect, test } = require('@playwright/test');
+const { visitStory } = require('../../test-utils/storybook');
+
+test.describe('AspectRatio @avt', () => {
+  test('@avt-default-state', async ({ page }) => {
+    await visitStory(page, {
+      component: 'AspectRatio',
+      id: 'components-aspectratio--default',
+      globals: {
+        theme: 'white',
+      },
+    });
+    await expect(page).toHaveNoACViolations('AspectRatio @avt-default-state');
+  });
+});
diff --git a/e2e/components/AspectRatio/AspectRatio-test.e2e.js b/e2e/components/AspectRatio/AspectRatio-test.e2e.js
@@ -7,9 +7,9 @@
 
 'use strict';
 
-const { expect, test } = require('@playwright/test');
+const { test } = require('@playwright/test');
 const { themes } = require('../../test-utils/env');
-const { snapshotStory, visitStory } = require('../../test-utils/storybook');
+const { snapshotStory } = require('../../test-utils/storybook');
 
 test.describe('AspectRatio', () => {
   themes.forEach((theme) => {
@@ -23,15 +23,4 @@ test.describe('AspectRatio', () => {
       });
     });
   });
-
-  test('accessibility-checker @avt', async ({ page }) => {
-    await visitStory(page, {
-      component: 'AspectRatio',
-      id: 'components-aspectratio--default',
-      globals: {
-        theme: 'white',
-      },
-    });
-    await expect(page).toHaveNoACViolations('AspectRatio');
-  });
 });
diff --git a/...nts/breadcrumb/breadcrumb-test.avt.e2e.js → ...nts/Breadcrumb/Breadcrumb-test.avt.e2e.js b/...nts/breadcrumb/breadcrumb-test.avt.e2e.js → ...nts/Breadcrumb/Breadcrumb-test.avt.e2e.js
@@ -19,7 +19,7 @@ test.describe('breadcrumb @avt', () => {
         theme: 'white',
       },
     });
-    await expect(page).toHaveNoACViolations('Breadcrumb');
+    await expect(page).toHaveNoACViolations('Breadcrumb-default-state');
   });
 
   test('@avt-advanced-states with overflow menu', async ({ page }) => {

diff --git a/...ponents/breadcrumb/breadcrumb-test.e2e.js → ...ponents/Breadcrumb/Breadcrumb-test.e2e.js b/...ponents/breadcrumb/breadcrumb-test.e2e.js → ...ponents/Breadcrumb/Breadcrumb-test.e2e.js
diff --git a/e2e/components/Button/Button-test.avt.e2e.js b/e2e/components/Button/Button-test.avt.e2e.js
@@ -11,7 +11,7 @@ const { expect, test } = require('@playwright/test');
 const { visitStory } = require('../../test-utils/storybook');
 
 test.describe('Button @avt', () => {
-  test('accessibility-checker', async ({ page }) => {
+  test('@avt-default-state', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--default',
@@ -22,7 +22,7 @@ test.describe('Button @avt', () => {
     await expect(page).toHaveNoACViolations('Button');
   });
 
-  test('accessibility-checker button danger', async ({ page }) => {
+  test('@avt-advanced-states danger', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--danger',
@@ -33,7 +33,7 @@ test.describe('Button @avt', () => {
     await expect(page).toHaveNoACViolations('Button-danger');
   });
 
-  test('accessibility-checker button ghost', async ({ page }) => {
+  test('@avt-advanced-states ghost', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--ghost',
@@ -44,7 +44,7 @@ test.describe('Button @avt', () => {
     await expect(page).toHaveNoACViolations('Button-ghost');
   });
 
-  test('accessibility-checker button icon-button', async ({ page }) => {
+  test('@avt-advanced-states icon-button', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--icon-button',
@@ -55,7 +55,7 @@ test.describe('Button @avt', () => {
     await expect(page).toHaveNoACViolations('Button-icon-button');
   });
 
-  test('accessibility-checker button secondary', async ({ page }) => {
+  test('@avt-advanced-states secondary', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--secondary',
@@ -66,7 +66,7 @@ test.describe('Button @avt', () => {
     await expect(page).toHaveNoACViolations('Button-secondary');
   });
 
-  test('accessibility-checker button set-of-buttons', async ({ page }) => {
+  test('@avt-advanced-states set-of-buttons', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--set-of-buttons',
@@ -77,7 +77,7 @@ test.describe('Button @avt', () => {
     await expect(page).toHaveNoACViolations('Button-set-of-buttons');
   });
 
-  test('accessibility-checker button skeleton', async ({ page }) => {
+  test('@avt-advanced-states skeleton', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--skeleton',
@@ -88,7 +88,7 @@ test.describe('Button @avt', () => {
     await expect(page).toHaveNoACViolations('Button-skeleton');
   });
 
-  test('accessibility-checker button tertiary', async ({ page }) => {
+  test('@avt-advanced-states tertiary', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--tertiary',
@@ -99,7 +99,7 @@ test.describe('Button @avt', () => {
     await expect(page).toHaveNoACViolations('Button-tertiary');
   });
 
-  test('accessibility-checker disabled button', async ({ page }) => {
+  test('@avt-advanced-states disabled', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--default',
@@ -115,7 +115,7 @@ test.describe('Button @avt', () => {
     await expect(page).toHaveNoACViolations('Button-disabled');
   });
 
-  test('accessibility-checker keyboard interaction', async ({ page }) => {
+  test('@avt-keyboard-nav', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--default',
@@ -128,7 +128,7 @@ test.describe('Button @avt', () => {
     await expect(page.getByRole('button')).toBeFocused();
   });
 
-  test('accessibility-checker mouse interaction', async ({ page }) => {
+  test('@avt-advanced-states mouse interaction', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--default',
@@ -141,7 +141,7 @@ test.describe('Button @avt', () => {
     await expect(page.getByRole('button')).toBeFocused();
   });
 
-  test('accessibility-checker hover state', async ({ page }) => {
+  test('@avt-advanced-states hover state', async ({ page }) => {
     await visitStory(page, {
       component: 'Button',
       id: 'components-button--default',

diff --git a/e2e/components/ClassPrefix/ClassPrefix-test.avt.e2e.js b/e2e/components/ClassPrefix/ClassPrefix-test.avt.e2e.js
@@ -0,0 +1,24 @@
+/**
+ * Copyright IBM Corp. 2016, 2023
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+const { expect, test } = require('@playwright/test');
+const { visitStory } = require('../../test-utils/storybook');
+
+test.describe('ClassPrefix @avt', () => {
+  test('@avt-default-state', async ({ page }) => {
+    await visitStory(page, {
+      component: 'ClassPrefix',
+      id: 'components-classprefix--default',
+      globals: {
+        theme: 'white',
+      },
+    });
+    await expect(page).toHaveNoACViolations('ClassPrefix @avt-default-state');
+  });
+});
diff --git a/e2e/components/ClassPrefix/ClassPrefix-test.e2e.js b/e2e/components/ClassPrefix/ClassPrefix-test.e2e.js
@@ -6,9 +6,9 @@
  */
 
 'use strict';
-const { expect, test } = require('@playwright/test');
+const { test } = require('@playwright/test');
 const { themes } = require('../../test-utils/env');
-const { snapshotStory, visitStory } = require('../../test-utils/storybook');
+const { snapshotStory } = require('../../test-utils/storybook');
 
 test.describe('ClassPrefix', () => {
   themes.forEach((theme) => {
@@ -22,15 +22,4 @@ test.describe('ClassPrefix', () => {
       });
     });
   });
-
-  test('accessibility-checker @avt', async ({ page }) => {
-    await visitStory(page, {
-      component: 'ClassPrefix',
-      id: 'components-classprefix--default',
-      globals: {
-        theme: 'white',
-      },
-    });
-    await expect(page).toHaveNoACViolations('ClassPrefix');
-  });
 });
diff --git a/e2e/components/CodeSnippet/CodeSnippet-test.avt.e2e.js b/e2e/components/CodeSnippet/CodeSnippet-test.avt.e2e.js
@@ -11,7 +11,7 @@ const { expect, test } = require('@playwright/test');
 const { visitStory } = require('../../test-utils/storybook');
 
 test.describe('CodeSnippet @avt', () => {
-  test('accessibility-checker CodeSnippet inline', async ({ page }) => {
+  test('@avt-default-state inline', async ({ page }) => {
     await visitStory(page, {
       component: 'CodeSnippet',
       id: 'components-codesnippet--inline',
@@ -22,9 +22,7 @@ test.describe('CodeSnippet @avt', () => {
     await expect(page).toHaveNoACViolations('CodeSnippet inline');
   });
 
-  test('accessibility-checker CodeSnippet inline hover @avt', async ({
-    page,
-  }) => {
+  test('@avt-advanced-states inline hover', async ({ page }) => {
     await visitStory(page, {
       component: 'CodeSnippet',
       id: 'components-codesnippet--inline',
@@ -38,9 +36,7 @@ test.describe('CodeSnippet @avt', () => {
     await expect(page).toHaveNoACViolations('CodeSnippet inline hover');
   });
 
-  test('accessibility-checker CodeSnippet inline focused @avt', async ({
-    page,
-  }) => {
+  test('@avt-advanced-states inline focused', async ({ page }) => {
     await visitStory(page, {
       component: 'CodeSnippet',
       id: 'components-codesnippet--inline',
@@ -54,7 +50,8 @@ test.describe('CodeSnippet @avt', () => {
     await expect(page).toHaveNoACViolations('CodeSnippet inline focused');
   });
 
-  test('accessibility-checker CodeSnippet multiline @avt', async ({ page }) => {
+  // Skipping due to a11y violation issue #14237
+  test.skip('@avt-default-state multiline', async ({ page }) => {
     await visitStory(page, {
       component: 'CodeSnippet',
       id: 'components-codesnippet--multiline',
@@ -65,9 +62,7 @@ test.describe('CodeSnippet @avt', () => {
     await expect(page).toHaveNoACViolations('CodeSnippet multiline');
   });
 
-  test('accessibility-checker CodeSnippet singleline @avt', async ({
-    page,
-  }) => {
+  test('@avt-default-state singleline', async ({ page }) => {
     await visitStory(page, {
       component: 'CodeSnippet',
       id: 'components-codesnippet--singleline',
@@ -78,7 +73,7 @@ test.describe('CodeSnippet @avt', () => {
     await expect(page).toHaveNoACViolations('CodeSnippet singleline');
   });
 
-  test('accessibility-checker CodeSnippet skeleton @avt', async ({ page }) => {
+  test('@avt-advanced-states skeleton', async ({ page }) => {
     await visitStory(page, {
       component: 'CodeSnippet',
       id: 'components-codesnippet--skeleton',
@@ -89,9 +84,7 @@ test.describe('CodeSnippet @avt', () => {
     await expect(page).toHaveNoACViolations('CodeSnippet skeleton');
   });
 
-  test('accessibility-checker CodeSnippet inline keyboard nav', async ({
-    page,
-  }) => {
+  test('inline @avt-keyboard-nav', async ({ page }) => {
     await visitStory(page, {
       component: 'CodeSnippet',
       id: 'components-codesnippet--inline',
@@ -110,9 +103,7 @@ test.describe('CodeSnippet @avt', () => {
     await expect(page.getByRole('tooltip')).toHaveText('Copied to clipboard');
   });
 
-  test('accessibility-checker CodeSnippet multiline keyboard nav', async ({
-    page,
-  }) => {
+  test('multiline @avt-keyboard-nav', async ({ page }) => {
     await visitStory(page, {
       component: 'CodeSnippet',
       id: 'components-codesnippet--multiline',
@@ -139,9 +130,7 @@ test.describe('CodeSnippet @avt', () => {
     await expect(page.getByText('Show more')).not.toBeVisible();
   });
 
-  test('accessibility-checker CodeSnippet singleline keyboard nav', async ({
-    page,
-  }) => {
+  test('singleline @avt-keyboard-nav', async ({ page }) => {
     await visitStory(page, {
       component: 'CodeSnippet',
       id: 'components-codesnippet--singleline',

diff --git a/e2e/components/ComboBox/ComboBox-test.avt.e2e.js b/e2e/components/ComboBox/ComboBox-test.avt.e2e.js
@@ -11,7 +11,7 @@ const { expect, test } = require('@playwright/test');
 const { visitStory } = require('../../test-utils/storybook');
 
 test.describe('ComboBox @avt', () => {
-  test('accessibility-checker combobox', async ({ page }) => {
+  test('@avt-default-state', async ({ page }) => {
     await visitStory(page, {
       component: 'ComboBox',
       id: 'components-combobox--default',
@@ -23,7 +23,7 @@ test.describe('ComboBox @avt', () => {
   });
 
   // Skipping now due to AVT violation, possible false positive
-  test.skip('accessibility-checker open combobox', async ({ page }) => {
+  test.skip('@avt-advanced-states open', async ({ page }) => {
     await visitStory(page, {
       component: 'ComboBox',
       id: 'components-combobox--default',
@@ -44,7 +44,7 @@ test.describe('ComboBox @avt', () => {
     await expect(page).toHaveNoACViolations('ComboBox-open');
   });
 
-  test('combobox - keyboard nav', async ({ page }) => {
+  test('@avt-keyboard-nav', async ({ page }) => {
     await visitStory(page, {
       component: 'ComboBox',
       id: 'components-combobox--default',