[Enterprise Search] Set up cypress-axe tests (#108465)

* Set up cypress-axe @see https://github.com/component-driven/cypress-axe * DRY out Kibana axe rules into constants that Cypress can use * Create shared & configured checkA11y command + fix string union type error + remove unnecessary tsconfig exclude * Add Overview plugin a11y tests * Add AS & WS placeholder a11y checks - Mostly just re-exporting the shared command and checking for failures, I only ran this after the shared axe config settings and found no failures * Configure our axe settings further to catch best practices - notably heading level issues (thanks Byron for catching this!) - however I now also need to set an ignore on a duplicate landmark violation caused by the global header (not sure why it's showing up - shouldn't it be out of context? bah) - remove option to pass args into checkA11y - I figure it's not super likely we'll need to override axe settings per-page (vs not running it), but we can pass it custom configs or args later if needed Co-authored-by: Kibana Machine <[email protected]>
elastic · Aug 24, 2021 · b27fb56 · b27fb56
1 parent 04103e9
commit b27fb56
Show file tree

Hide file tree

Showing 12 changed files with 113 additions and 55 deletions.
diff --git a/package.json b/package.json
@@ -687,6 +687,7 @@
     "css-loader": "^3.4.2",
     "cssnano": "^4.1.11",
     "cypress": "^6.8.0",
+    "cypress-axe": "^0.13.0",
     "cypress-cucumber-preprocessor": "^2.5.2",
     "cypress-multi-reporters": "^1.4.0",
     "cypress-pipe": "^2.0.0",

diff --git a/test/accessibility/services/a11y/a11y.ts b/test/accessibility/services/a11y/a11y.ts
@@ -10,6 +10,7 @@ import chalk from 'chalk';
 import testSubjectToCss from '@kbn/test-subj-selector';
 
 import { FtrService } from '../../ftr_provider_context';
+import { AXE_CONFIG, AXE_OPTIONS } from './constants';
 import { AxeReport, printResult } from './axe_report';
 // @ts-ignore JS that is run in browser as is
 import { analyzeWithAxe, analyzeWithAxeWithClient } from './analyze_with_axe';
@@ -77,34 +78,21 @@ export class AccessibilityService extends FtrService {
   }
 
   private async captureAxeReport(context: AxeContext): Promise<AxeReport> {
-    const axeOptions = {
-      reporter: 'v2',
-      runOnly: ['wcag2a', 'wcag2aa'],
-      rules: {
-        'color-contrast': {
-          enabled: false, // disabled because we have too many failures
-        },
-        bypass: {
-          enabled: false, // disabled because it's too flaky
-        },
-      },
-    };
-
     await this.Wd.driver.manage().setTimeouts({
       ...(await this.Wd.driver.manage().getTimeouts()),
       script: 600000,
     });
 
     const report = normalizeResult(
-      await this.browser.executeAsync(analyzeWithAxe, context, axeOptions)
+      await this.browser.executeAsync(analyzeWithAxe, context, AXE_CONFIG, AXE_OPTIONS)
     );
 
     if (report !== false) {
       return report;
     }
 
     const withClientReport = normalizeResult(
-      await this.browser.executeAsync(analyzeWithAxeWithClient, context, axeOptions)
+      await this.browser.executeAsync(analyzeWithAxeWithClient, context, AXE_CONFIG, AXE_OPTIONS)
     );
 
     if (withClientReport === false) {

diff --git a/test/accessibility/services/a11y/analyze_with_axe.js b/test/accessibility/services/a11y/analyze_with_axe.js
@@ -8,45 +8,11 @@
 
 import { readFileSync } from 'fs';
 
-export function analyzeWithAxe(context, options, callback) {
+export function analyzeWithAxe(context, config, options, callback) {
   Promise.resolve()
     .then(() => {
       if (window.axe) {
-        window.axe.configure({
-          rules: [
-            {
-              id: 'scrollable-region-focusable',
-              selector: '[data-skip-axe="scrollable-region-focusable"]',
-            },
-            {
-              id: 'aria-required-children',
-              selector: '[data-skip-axe="aria-required-children"] > *',
-            },
-            {
-              id: 'label',
-              selector: '[data-test-subj="comboBoxSearchInput"] *',
-            },
-            {
-              id: 'aria-roles',
-              selector: '[data-test-subj="comboBoxSearchInput"] *',
-            },
-            {
-              // EUI bug: https://github.com/elastic/eui/issues/4474
-              id: 'aria-required-parent',
-              selector: '[class=*"euiDataGridRowCell"][role="gridcell"]',
-            },
-            {
-              // 3rd-party library; button has aria-describedby
-              id: 'button-name',
-              selector: '[data-rbd-drag-handle-draggable-id]',
-            },
-            {
-              // EUI bug: https://github.com/elastic/eui/issues/4536
-              id: 'duplicate-id',
-              selector: '.euiSuperDatePicker *',
-            },
-          ],
-        });
+        window.axe.configure(config);
         return window.axe.run(context, options);
       }
 

diff --git a/test/accessibility/services/a11y/constants.ts b/test/accessibility/services/a11y/constants.ts
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ReporterVersion } from 'axe-core';
+
+export const AXE_CONFIG = {
+  rules: [
+    {
+      id: 'scrollable-region-focusable',
+      selector: '[data-skip-axe="scrollable-region-focusable"]',
+    },
+    {
+      id: 'aria-required-children',
+      selector: '[data-skip-axe="aria-required-children"] > *',
+    },
+    {
+      id: 'label',
+      selector: '[data-test-subj="comboBoxSearchInput"] *',
+    },
+    {
+      id: 'aria-roles',
+      selector: '[data-test-subj="comboBoxSearchInput"] *',
+    },
+    {
+      // EUI bug: https://github.com/elastic/eui/issues/4474
+      id: 'aria-required-parent',
+      selector: '[class=*"euiDataGridRowCell"][role="gridcell"]',
+    },
+    {
+      // 3rd-party library; button has aria-describedby
+      id: 'button-name',
+      selector: '[data-rbd-drag-handle-draggable-id]',
+    },
+    {
+      // EUI bug: https://github.com/elastic/eui/issues/4536
+      id: 'duplicate-id',
+      selector: '.euiSuperDatePicker *',
+    },
+  ],
+};
+
+export const AXE_OPTIONS = {
+  reporter: 'v2' as ReporterVersion,
+  runOnly: ['wcag2a', 'wcag2aa'],
+  rules: {
+    'color-contrast': {
+      enabled: false, // disabled because we have too many failures
+    },
+    bypass: {
+      enabled: false, // disabled because it's too flaky
+    },
+  },
+};
diff --git a/...gins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts b/...gins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { login } from '../support/commands';
+import { login, checkA11y } from '../support/commands';
 
 context('Engines', () => {
   beforeEach(() => {
@@ -14,5 +14,6 @@ context('Engines', () => {
 
   it('renders', () => {
     cy.contains('Engines');
+    checkA11y();
   });
 });
diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/support/commands.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/support/commands.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+export { checkA11y } from '../../../shared/cypress/commands';
 import { login as baseLogin } from '../../../shared/cypress/commands';
 import { appSearchPath } from '../../../shared/cypress/routes';
 

diff --git a/...erprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts b/...erprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { login } from '../../../shared/cypress/commands';
+import { login, checkA11y } from '../../../shared/cypress/commands';
 import { overviewPath } from '../../../shared/cypress/routes';
 
 context('Enterprise Search Overview', () => {
@@ -26,6 +26,8 @@ context('Enterprise Search Overview', () => {
       .contains('Open Workplace Search')
       .should('have.attr', 'href')
       .and('match', /workplace_search/);
+
+    checkA11y();
   });
 
   it('should have a setup guide', () => {
@@ -38,5 +40,7 @@ context('Enterprise Search Overview', () => {
     cy.visit(`${overviewPath}/setup_guide`);
     cy.contains('Setup Guide');
     cy.contains('Add your Enterprise Search host URL to your Kibana configuration');
+
+    checkA11y();
   });
 });
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts
@@ -33,3 +33,34 @@ export const login = ({
     },
   });
 };
+
+/*
+ * Cypress setup/helpers
+ */
+
+// eslint-disable-next-line import/no-extraneous-dependencies
+import 'cypress-axe'; // eslint complains this should be in `dependencies` and not `devDependencies`, but these tests should only run on dev
+import { AXE_CONFIG, AXE_OPTIONS } from 'test/accessibility/services/a11y/constants';
+
+const axeConfig = {
+  ...AXE_CONFIG,
+  rules: [
+    ...AXE_CONFIG.rules,
+    {
+      id: 'landmark-no-duplicate-banner',
+      selector: '[data-test-subj="headerGlobalNav"]',
+    },
+  ],
+};
+const axeOptions = {
+  ...AXE_OPTIONS,
+  runOnly: [...AXE_OPTIONS.runOnly, 'best-practice'],
+};
+
+// @see https://github.com/component-driven/cypress-axe#cychecka11y for params
+export const checkA11y = () => {
+  cy.injectAxe();
+  cy.configureAxe(axeConfig);
+  const context = '.kbnAppWrapper'; // Scopes a11y checks to only our app
+  cy.checkA11y(context, axeOptions);
+};
diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json
@@ -1,8 +1,9 @@
 {
   "extends": "../../../../../../../tsconfig.base.json",
+  "references": [{ "path": "../../../../../../../test/tsconfig.json" }],
   "include": ["./**/*"],
   "compilerOptions": {
     "outDir": "../../../../target/cypress/types/shared",
-    "types": ["cypress", "node"]
+    "types": ["cypress", "cypress-axe", "node"]
   }
 }
diff --git a/...terprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts b/...terprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { login } from '../support/commands';
+import { login, checkA11y } from '../support/commands';
 
 context('Overview', () => {
   beforeEach(() => {
@@ -14,5 +14,6 @@ context('Overview', () => {
 
   it('renders', () => {
     cy.contains('Workplace Search');
+    checkA11y();
   });
 });
diff --git a/...lugins/enterprise_search/public/applications/workplace_search/cypress/support/commands.ts b/...lugins/enterprise_search/public/applications/workplace_search/cypress/support/commands.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+export { checkA11y } from '../../../shared/cypress/commands';
 import { login as baseLogin } from '../../../shared/cypress/commands';
 import { workplaceSearchPath } from '../../../shared/cypress/routes';
 

diff --git a/yarn.lock b/yarn.lock
@@ -10939,6 +10939,11 @@ cyclist@~0.2.2:
   resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-0.2.2.tgz#1b33792e11e914a2fd6d6ed6447464444e5fa640"
   integrity sha1-GzN5LhHpFKL9bW7WRHRkRE5fpkA=
 
+cypress-axe@^0.13.0:
+  version "0.13.0"
+  resolved "https://registry.yarnpkg.com/cypress-axe/-/cypress-axe-0.13.0.tgz#3234e1a79a27701f2451fcf2f333eb74204c7966"
+  integrity sha512-fCIy7RiDCm7t30U3C99gGwQrUO307EYE1QqXNaf9ToK4DVqW8y5on+0a/kUHMrHdlls2rENF6TN9ZPpPpwLrnw==
+
 cypress-cucumber-preprocessor@^2.5.2:
   version "2.5.5"
   resolved "https://registry.yarnpkg.com/cypress-cucumber-preprocessor/-/cypress-cucumber-preprocessor-2.5.5.tgz#af20aa40d3dd6dc67b28f6819411831bb0bea925"
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,7 @@ @@
      * 2.0.
      */
+    export { checkA11y } from '../../../shared/cypress/commands';
     import { login as baseLogin } from '../../../shared/cypress/commands';
     import { appSearchPath } from '../../../shared/cypress/routes';
@@ Expand Down @@