From faff2b9c7096620d9a2a5aac75b95bc8549f35cf Mon Sep 17 00:00:00 2001 From: Constance Chen Date: Thu, 12 Aug 2021 13:54:12 -0700 Subject: [PATCH 1/6] Set up cypress-axe @see https://github.com/component-driven/cypress-axe --- package.json | 1 + .../public/applications/shared/cypress/tsconfig.json | 2 +- yarn.lock | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 950a916b2419f..bfef60098cd30 100644 --- a/package.json +++ b/package.json @@ -689,6 +689,7 @@ "css-loader": "^3.4.2", "cssnano": "^4.1.11", "cypress": "^6.8.0", + "cypress-axe": "^0.13.0", "cypress-cucumber-preprocessor": "^2.5.2", "cypress-multi-reporters": "^1.4.0", "cypress-pipe": "^2.0.0", diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json index be8ccac5f5e72..74453560a3f25 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json +++ b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json @@ -3,6 +3,6 @@ "include": ["./**/*"], "compilerOptions": { "outDir": "../../../../target/cypress/types/shared", - "types": ["cypress", "node"] + "types": ["cypress", "cypress-axe", "node"] } } diff --git a/yarn.lock b/yarn.lock index 121ba05364a81..a1a59b1478b85 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10916,6 +10916,11 @@ cyclist@~0.2.2: resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-0.2.2.tgz#1b33792e11e914a2fd6d6ed6447464444e5fa640" integrity sha1-GzN5LhHpFKL9bW7WRHRkRE5fpkA= +cypress-axe@^0.13.0: + version "0.13.0" + resolved "https://registry.yarnpkg.com/cypress-axe/-/cypress-axe-0.13.0.tgz#3234e1a79a27701f2451fcf2f333eb74204c7966" + integrity sha512-fCIy7RiDCm7t30U3C99gGwQrUO307EYE1QqXNaf9ToK4DVqW8y5on+0a/kUHMrHdlls2rENF6TN9ZPpPpwLrnw== + cypress-cucumber-preprocessor@^2.5.2: version "2.5.5" resolved "https://registry.yarnpkg.com/cypress-cucumber-preprocessor/-/cypress-cucumber-preprocessor-2.5.5.tgz#af20aa40d3dd6dc67b28f6819411831bb0bea925" From acf5d5f875f5764e99852da4ad261d44043910ab Mon Sep 17 00:00:00 2001 From: Constance Chen Date: Thu, 12 Aug 2021 14:02:40 -0700 Subject: [PATCH 2/6] DRY out Kibana axe rules into constants that Cypress can use --- test/accessibility/services/a11y/a11y.ts | 18 +----- .../services/a11y/analyze_with_axe.js | 38 +------------ test/accessibility/services/a11y/constants.ts | 56 +++++++++++++++++++ 3 files changed, 61 insertions(+), 51 deletions(-) create mode 100644 test/accessibility/services/a11y/constants.ts diff --git a/test/accessibility/services/a11y/a11y.ts b/test/accessibility/services/a11y/a11y.ts index 4b01b0dd3b953..f4d5ceba5a6e3 100644 --- a/test/accessibility/services/a11y/a11y.ts +++ b/test/accessibility/services/a11y/a11y.ts @@ -10,6 +10,7 @@ import chalk from 'chalk'; import testSubjectToCss from '@kbn/test-subj-selector'; import { FtrService } from '../../ftr_provider_context'; +import { AXE_CONFIG, AXE_OPTIONS } from './constants'; import { AxeReport, printResult } from './axe_report'; // @ts-ignore JS that is run in browser as is import { analyzeWithAxe, analyzeWithAxeWithClient } from './analyze_with_axe'; @@ -77,26 +78,13 @@ export class AccessibilityService extends FtrService { } private async captureAxeReport(context: AxeContext): Promise { - const axeOptions = { - reporter: 'v2', - runOnly: ['wcag2a', 'wcag2aa'], - rules: { - 'color-contrast': { - enabled: false, // disabled because we have too many failures - }, - bypass: { - enabled: false, // disabled because it's too flaky - }, - }, - }; - await this.Wd.driver.manage().setTimeouts({ ...(await this.Wd.driver.manage().getTimeouts()), script: 600000, }); const report = normalizeResult( - await this.browser.executeAsync(analyzeWithAxe, context, axeOptions) + await this.browser.executeAsync(analyzeWithAxe, context, AXE_CONFIG, AXE_OPTIONS) ); if (report !== false) { @@ -104,7 +92,7 @@ export class AccessibilityService extends FtrService { } const withClientReport = normalizeResult( - await this.browser.executeAsync(analyzeWithAxeWithClient, context, axeOptions) + await this.browser.executeAsync(analyzeWithAxeWithClient, context, AXE_CONFIG, AXE_OPTIONS) ); if (withClientReport === false) { diff --git a/test/accessibility/services/a11y/analyze_with_axe.js b/test/accessibility/services/a11y/analyze_with_axe.js index 4bd29dbab7efc..6e38e7f6f751f 100644 --- a/test/accessibility/services/a11y/analyze_with_axe.js +++ b/test/accessibility/services/a11y/analyze_with_axe.js @@ -8,45 +8,11 @@ import { readFileSync } from 'fs'; -export function analyzeWithAxe(context, options, callback) { +export function analyzeWithAxe(context, config, options, callback) { Promise.resolve() .then(() => { if (window.axe) { - window.axe.configure({ - rules: [ - { - id: 'scrollable-region-focusable', - selector: '[data-skip-axe="scrollable-region-focusable"]', - }, - { - id: 'aria-required-children', - selector: '[data-skip-axe="aria-required-children"] > *', - }, - { - id: 'label', - selector: '[data-test-subj="comboBoxSearchInput"] *', - }, - { - id: 'aria-roles', - selector: '[data-test-subj="comboBoxSearchInput"] *', - }, - { - // EUI bug: https://github.com/elastic/eui/issues/4474 - id: 'aria-required-parent', - selector: '[class=*"euiDataGridRowCell"][role="gridcell"]', - }, - { - // 3rd-party library; button has aria-describedby - id: 'button-name', - selector: '[data-rbd-drag-handle-draggable-id]', - }, - { - // EUI bug: https://github.com/elastic/eui/issues/4536 - id: 'duplicate-id', - selector: '.euiSuperDatePicker *', - }, - ], - }); + window.axe.configure(config); return window.axe.run(context, options); } diff --git a/test/accessibility/services/a11y/constants.ts b/test/accessibility/services/a11y/constants.ts new file mode 100644 index 0000000000000..8fcce1e566ad8 --- /dev/null +++ b/test/accessibility/services/a11y/constants.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const AXE_CONFIG = { + rules: [ + { + id: 'scrollable-region-focusable', + selector: '[data-skip-axe="scrollable-region-focusable"]', + }, + { + id: 'aria-required-children', + selector: '[data-skip-axe="aria-required-children"] > *', + }, + { + id: 'label', + selector: '[data-test-subj="comboBoxSearchInput"] *', + }, + { + id: 'aria-roles', + selector: '[data-test-subj="comboBoxSearchInput"] *', + }, + { + // EUI bug: https://github.com/elastic/eui/issues/4474 + id: 'aria-required-parent', + selector: '[class=*"euiDataGridRowCell"][role="gridcell"]', + }, + { + // 3rd-party library; button has aria-describedby + id: 'button-name', + selector: '[data-rbd-drag-handle-draggable-id]', + }, + { + // EUI bug: https://github.com/elastic/eui/issues/4536 + id: 'duplicate-id', + selector: '.euiSuperDatePicker *', + }, + ], +}; + +export const AXE_OPTIONS = { + reporter: 'v2', + runOnly: ['wcag2a', 'wcag2aa'], + rules: { + 'color-contrast': { + enabled: false, // disabled because we have too many failures + }, + bypass: { + enabled: false, // disabled because it's too flaky + }, + }, +}; From dcb73553ed20f178fdccb421168cf7b9b51245c4 Mon Sep 17 00:00:00 2001 From: Constance Chen Date: Thu, 12 Aug 2021 14:08:49 -0700 Subject: [PATCH 3/6] Create shared & configured checkA11y command + fix string union type error + remove unnecessary tsconfig exclude --- test/accessibility/services/a11y/constants.ts | 4 +++- .../applications/shared/cypress/commands.ts | 16 ++++++++++++++++ .../applications/shared/cypress/tsconfig.json | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/test/accessibility/services/a11y/constants.ts b/test/accessibility/services/a11y/constants.ts index 8fcce1e566ad8..e5f6773f03502 100644 --- a/test/accessibility/services/a11y/constants.ts +++ b/test/accessibility/services/a11y/constants.ts @@ -6,6 +6,8 @@ * Side Public License, v 1. */ +import { ReporterVersion } from 'axe-core'; + export const AXE_CONFIG = { rules: [ { @@ -43,7 +45,7 @@ export const AXE_CONFIG = { }; export const AXE_OPTIONS = { - reporter: 'v2', + reporter: 'v2' as ReporterVersion, runOnly: ['wcag2a', 'wcag2aa'], rules: { 'color-contrast': { diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts index 5f9738fae5064..78179db2f5a50 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts @@ -33,3 +33,19 @@ export const login = ({ }, }); }; + +/* + * Cypress setup/helpers + */ + +// eslint-disable-next-line import/no-extraneous-dependencies +import 'cypress-axe'; // eslint complains this should be in `dependencies` and not `devDependencies`, but these tests should only run on dev +import { AXE_CONFIG, AXE_OPTIONS } from 'test/accessibility/services/a11y/constants'; + +// @see https://github.com/component-driven/cypress-axe#cychecka11y for params +export const checkA11y = ({ ...args } = {}) => { + cy.injectAxe(); + cy.configureAxe(AXE_CONFIG); + const context = '.kbnAppWrapper'; // Scopes a11y checks to only our app + cy.checkA11y(context, { ...AXE_OPTIONS, ...args }); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json index 74453560a3f25..e728943de044e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json +++ b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/tsconfig.json @@ -1,5 +1,6 @@ { "extends": "../../../../../../../tsconfig.base.json", + "references": [{ "path": "../../../../../../../test/tsconfig.json" }], "include": ["./**/*"], "compilerOptions": { "outDir": "../../../../target/cypress/types/shared", From 450f74f82b344f045a333f41611fc6182ae223f3 Mon Sep 17 00:00:00 2001 From: Constance Chen Date: Thu, 12 Aug 2021 14:14:19 -0700 Subject: [PATCH 4/6] Add Overview plugin a11y tests --- .../enterprise_search/cypress/integration/overview.spec.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts index 4c9a159a6736f..45bd8f68a85fb 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search/cypress/integration/overview.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { login } from '../../../shared/cypress/commands'; +import { login, checkA11y } from '../../../shared/cypress/commands'; import { overviewPath } from '../../../shared/cypress/routes'; context('Enterprise Search Overview', () => { @@ -26,6 +26,8 @@ context('Enterprise Search Overview', () => { .contains('Open Workplace Search') .should('have.attr', 'href') .and('match', /workplace_search/); + + checkA11y(); }); it('should have a setup guide', () => { @@ -38,5 +40,7 @@ context('Enterprise Search Overview', () => { cy.visit(`${overviewPath}/setup_guide`); cy.contains('Setup Guide'); cy.contains('Add your Enterprise Search host URL to your Kibana configuration'); + + checkA11y(); }); }); From 1952a1c33b1d830a2149668ddb303be2df5374cf Mon Sep 17 00:00:00 2001 From: Constance Chen Date: Thu, 12 Aug 2021 14:15:04 -0700 Subject: [PATCH 5/6] Add AS & WS placeholder a11y checks - Mostly just re-exporting the shared command and checking for failures, I only ran this after the shared axe config settings and found no failures --- .../app_search/cypress/integration/engines.spec.ts | 3 ++- .../public/applications/app_search/cypress/support/commands.ts | 1 + .../workplace_search/cypress/integration/overview.spec.ts | 3 ++- .../applications/workplace_search/cypress/support/commands.ts | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts index 5e651aab075c6..c57518a55cb1a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/integration/engines.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { login } from '../support/commands'; +import { login, checkA11y } from '../support/commands'; context('Engines', () => { beforeEach(() => { @@ -14,5 +14,6 @@ context('Engines', () => { it('renders', () => { cy.contains('Engines'); + checkA11y(); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/support/commands.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/support/commands.ts index 50b5fcd179297..9c60d044aa21a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/support/commands.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/cypress/support/commands.ts @@ -5,6 +5,7 @@ * 2.0. */ +export { checkA11y } from '../../../shared/cypress/commands'; import { login as baseLogin } from '../../../shared/cypress/commands'; import { appSearchPath } from '../../../shared/cypress/routes'; diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts index 8ce6e4ebcfb05..9610cf0d25b85 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/integration/overview.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { login } from '../support/commands'; +import { login, checkA11y } from '../support/commands'; context('Overview', () => { beforeEach(() => { @@ -14,5 +14,6 @@ context('Overview', () => { it('renders', () => { cy.contains('Workplace Search'); + checkA11y(); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/support/commands.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/support/commands.ts index d91b73fd78c05..3b73d4cefa971 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/support/commands.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress/support/commands.ts @@ -5,6 +5,7 @@ * 2.0. */ +export { checkA11y } from '../../../shared/cypress/commands'; import { login as baseLogin } from '../../../shared/cypress/commands'; import { workplaceSearchPath } from '../../../shared/cypress/routes'; From 722f8278599d219edd351f1c1ae642302c8fc962 Mon Sep 17 00:00:00 2001 From: Constance Chen Date: Fri, 13 Aug 2021 11:07:48 -0700 Subject: [PATCH 6/6] Configure our axe settings further to catch best practices - notably heading level issues (thanks Byron for catching this!) - however I now also need to set an ignore on a duplicate landmark violation caused by the global header (not sure why it's showing up - shouldn't it be out of context? bah) - remove option to pass args into checkA11y - I figure it's not super likely we'll need to override axe settings per-page (vs not running it), but we can pass it custom configs or args later if needed --- .../applications/shared/cypress/commands.ts | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts index 78179db2f5a50..475343948f348 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/cypress/commands.ts @@ -42,10 +42,25 @@ export const login = ({ import 'cypress-axe'; // eslint complains this should be in `dependencies` and not `devDependencies`, but these tests should only run on dev import { AXE_CONFIG, AXE_OPTIONS } from 'test/accessibility/services/a11y/constants'; +const axeConfig = { + ...AXE_CONFIG, + rules: [ + ...AXE_CONFIG.rules, + { + id: 'landmark-no-duplicate-banner', + selector: '[data-test-subj="headerGlobalNav"]', + }, + ], +}; +const axeOptions = { + ...AXE_OPTIONS, + runOnly: [...AXE_OPTIONS.runOnly, 'best-practice'], +}; + // @see https://github.com/component-driven/cypress-axe#cychecka11y for params -export const checkA11y = ({ ...args } = {}) => { +export const checkA11y = () => { cy.injectAxe(); - cy.configureAxe(AXE_CONFIG); + cy.configureAxe(axeConfig); const context = '.kbnAppWrapper'; // Scopes a11y checks to only our app - cy.checkA11y(context, { ...AXE_OPTIONS, ...args }); + cy.checkA11y(context, axeOptions); };