[terra-functional-testing] Add Accessibility Reporter

packages/terra-functional-testing/package.json

@@ -50,9 +50,11 @@
     "@wdio/local-runner": "^6.1.4",
     "@wdio/logger": "^6.0.16",
     "@wdio/mocha-framework": "^6.1.0",
+    "@wdio/reporter": "^6.6.6",
     "@wdio/spec-reporter": "^6.0.16",
     "@wdio/sync": "^6.1.14",
     "axe-core": "4.0.2",
+    "chalk": "^4.1.0",
     "commander": "^5.1.0",
     "expect": "^26.4.2",
     "express": "^4.17.1",

packages/terra-functional-testing/src/commands/axe/inject.js

@@ -4,11 +4,19 @@
  * Injects axe-core into the browser.
  * @param {Object} options - The axe configuration options.
  */
-const injectAxe = (options) => {
+const injectAxe = (options = {}) => {
   // eslint-disable-next-line global-require
   const { source } = require('axe-core/axe.min.js');
 
-  browser.execute(`${source}\n ${options ? `axe.configure(${JSON.stringify(options)})` : ''}`);
+  const { rules } = options;
+
+  const config = {
+    ...options,
+    // axe.configure has a different API than axe.run. Rules must be an array for axe.configure.
+    ...rules && { rules: Object.keys(rules).map((rule) => ({ ...rules[rule], id: rule })) },
+  };
+
+  browser.execute(`${source}\n axe.configure(${JSON.stringify(config)})`);
 };
 
 module.exports = injectAxe;

packages/terra-functional-testing/src/commands/axe/run.js

@@ -12,28 +12,32 @@ const runAxe = (overrides = {}) => {
     typeof service === 'function' && service.name === 'TerraService'
   ));
 
-  const { axe: axeOptions } = options;
+  const { axe: axeOptions = {} } = options;
+  const { rules: globalRules = {} } = axeOptions;
+
+  const globalRuleOverrides = {
+    /**
+     * This rule was introduced in axe-core v3.3 and causes failures in many Terra components.
+     * The solution to address this failure vary by component. It is being disabled until a solution is identified in the future.
+     *
+     * Reference: https://github.com/cerner/terra-framework/issues/991
+     */
+    'scrollable-region-focusable': { enabled: false },
+    /**
+     * Rules configured through the Terra Service axe options are applied globally.
+     */
+    ...globalRules,
+  };
+
   const isAxeUnavailable = browser.execute(() => window.axe === undefined);
 
   // Inject axe-core onto the page if it has not already been initialized.
   if (isAxeUnavailable) {
-    injectAxe(axeOptions);
+    injectAxe({ ...axeOptions, rules: globalRuleOverrides });
   }
 
-  /**
-    * This rule was introduced in axe-core v3.3 and causes failures in many Terra components.
-    * The solution to address this failure vary by component. It is being disabled until a solution is identified in the future.
-    *
-    * Reference: https://github.com/cerner/terra-framework/issues/991
-    */
-  const ruleOverrides = {
-    'scrollable-region-focusable': { enabled: false },
-  };
-
-  // Merge the global rules and overrides together.
   const rules = {
-    ...ruleOverrides,
-    ...axeOptions && axeOptions.rules,
+    ...globalRuleOverrides,
     ...overrides.rules,
   };
 
@@ -43,7 +47,7 @@ const runAxe = (overrides = {}) => {
     axe.run(document, opts, function (error, result) {
       done({ error, result });
     });
-  }, { rules, restoreScroll: true, runOnly: ['wcag2a', 'wcag2aa', 'wcag21aa', 'section508'] });
+  }, { rules, runOnly: ['wcag2a', 'wcag2aa', 'wcag21aa', 'section508'] });
 };
 
 module.exports = runAxe;

packages/terra-functional-testing/src/commands/expect/toBeAccessible.js

@@ -8,7 +8,13 @@ const { runAxe } = require('../axe');
  */
 function toBeAccessible(_, options = {}) {
   const { result } = runAxe(options);
-  const { violations } = result;
+  const { incomplete, violations } = result;
+
+  // Rules that fail but are marked for review are returned in the incomplete array.
+  // https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#results-object
+  if (incomplete && incomplete.length > 0) {
+    process.emit('terra:report:accessibility', { incomplete });
+  }
 
   return {
     pass: violations.length === 0,

packages/terra-functional-testing/src/config/wdio.conf.js

@@ -5,6 +5,7 @@ const ip = require('ip');
 const SeleniumDockerService = require('../services/wdio-selenium-docker-service');
 const TerraService = require('../services/wdio-terra-service');
 const AssetServerService = require('../services/wdio-asset-server-service');
+const AccessibilityReporter = require('../reporters/wdio-accessibility-reporter');
 
 const {
   LOCALE,
@@ -157,7 +158,7 @@ exports.config = {
   // Test reporter for stdout.
   // The only one supported by default is 'dot'
   // see also: https://webdriver.io/docs/dot-reporter.html
-  reporters: ['spec'],
+  reporters: ['spec', AccessibilityReporter],
   //
   // Options to be passed to Mocha.
   // See the full list at http://mochajs.org/

packages/terra-functional-testing/src/reporters/wdio-accessibility-reporter.js

@@ -0,0 +1,123 @@
+const chalk = require('chalk');
+const WDIOReporter = require('@wdio/reporter').default;
+
+class AccessibilityReporter extends WDIOReporter {
+  /**
+   * Indents a string.
+   * @param {string} string - The string to indent.
+   * @param {number} indent - The number of spaces to indent the string.
+   */
+  static indent(string, indent) {
+    if (indent > 0) {
+      return string.replace(/^(?!\s*$)/gm, Array(indent + 1).join(' '));
+    }
+
+    return string;
+  }
+
+  constructor(options) {
+    super(options);
+
+    this.currentTest = null;
+    this.accessibilityResults = {};
+
+    process.on('terra:report:accessibility', this.onReportAccessibility.bind(this));
+  }
+
+  /**
+   * Hook invoked at the start of the test.
+   * @param {TestStats} test - The current test object.
+   */
+  onTestStart(test) {
+    this.currentTest = test.uid;
+  }
+
+  /**
+   * Hook invoked when accessibility results are reported.
+   * @param {Object} results - Accessibility results.
+   * @param {array} results.incomplete - Incomplete accessibility results.
+   */
+  onReportAccessibility(results) {
+    this.accessibilityResults[this.currentTest] = results;
+  }
+
+  /**
+   * Hook invoked when the runner ends.
+   * @param RunnerStats runner - The test runner stats object.
+   */
+  onRunnerEnd(runner) {
+    this.printReport(runner);
+  }
+
+  /**
+   * Print the accessibility report.
+   * @param {RunnerStats} runner - The test runner.
+   */
+  printReport(runner) {
+    const warnings = Object.keys(this.accessibilityResults);
+
+    if (warnings.length === 0) {
+      return;
+    }
+
+    const rootSuite = this.currentSuites.find((suite) => suite.uid === '(root)');
+    const output = this.travelSuite(rootSuite);
+    const spec = runner.specs[0];
+
+    this.write(`Spec: ${spec}\n${output}`);
+  }
+
+  /**
+   * Travels the suite tree to generate the accessibility report.
+   * @param {SuiteStats} currentSuite - The current suite.
+   * @param {number} depth - The current nesting depth of the suite tree.
+   * @returns {string} - An accessibility report.
+   */
+  travelSuite(currentSuite, depth = 0) {
+    const { suites, tests, title } = currentSuite;
+
+    const warnings = [];
+
+    suites.forEach((suite) => {
+      const output = this.travelSuite(suite, depth + 1);
+
+      if (output.length > 0) {
+        warnings.push(output);
+      }
+    });
+
+    tests.forEach((test) => {
+      const { uid } = test;
+
+      if (this.accessibilityResults[uid]) {
+        const testTitle = this.formatTestWarning(uid, (depth * 2) + 2);
+
+        warnings.push(testTitle);
+      }
+    });
+
+    if (warnings.length > 0) {
+      const suiteTitle = title === '(root)' ? '' : title;
+
+      return `${AccessibilityReporter.indent(suiteTitle, (depth * 2))}\n${warnings.join('\n')}\n`;
+    }
+
+    return '';
+  }
+
+  /**
+   * Formats an accessibility warning summary for a test.
+   * @param {string} uid - The unique identifier of the test.
+   * @param {number} indent - The indentation the report should have,
+   */
+  formatTestWarning(uid, indent) {
+    const { title } = this.tests[uid];
+    const { incomplete } = this.accessibilityResults[uid];
+
+    const string = `${chalk.yellow('warning')} ${title}\n\n ${chalk.yellow(`${JSON.stringify(incomplete, null, 2)}`)}`;
+
+    return AccessibilityReporter.indent(string, indent);
+  }
+}
+
+module.exports = AccessibilityReporter;

packages/terra-functional-testing/tests/fixtures/reporters/test-data-1.json

@@ -0,0 +1,75 @@
+{
+  "title": "(root)",
+  "suites": [
+    {
+      "type": "suite:start",
+      "start": "2020-10-27T01:14:33.430Z",
+      "_duration": 214,
+      "uid": "suite-0-0",
+      "cid": "0-0",
+      "title": "Example Describe 1",
+      "fullTitle": "Example Describe 1",
+      "tests": [],
+      "hooks": [],
+      "suites": [
+        {
+          "type": "suite:start",
+          "start": "2020-10-27T01:14:33.430Z",
+          "_duration": 213,
+          "uid": "suite-1-0",
+          "cid": "0-0",
+          "title": "Example Describe 2",
+          "fullTitle": "Example Describe 1 Example Describe 2",
+          "tests": [
+            {
+              "type": "test",
+              "start": "2020-10-27T01:14:33.431Z",
+              "_duration": 101,
+              "uid": "test-10-0",
+              "cid": "0-0",
+              "title": "example it 1",
+              "fullTitle": "Example Describe 1 Example Describe 2 example it 1",
+              "output": [],
+              "retries": 0,
+              "state": "passed",
+              "end": "2020-10-27T01:14:33.532Z"
+            },
+            {
+              "type": "test",
+              "start": "2020-10-27T01:14:33.532Z",
+              "_duration": 53,
+              "uid": "test-10-1",
+              "cid": "0-0",
+              "title": "example it 2",
+              "fullTitle": "Example Describe 1 Example Describe 2 example it 2",
+              "output": [],
+              "retries": 0,
+              "state": "passed",
+              "end": "2020-10-27T01:14:33.585Z"
+            },
+            {
+              "type": "test",
+              "start": "2020-10-27T01:14:33.586Z",
+              "_duration": 57,
+              "uid": "test-10-2",
+              "cid": "0-0",
+              "title": "example it 3",
+              "fullTitle": "Example Describe 1 Example Describe 2 example it 3",
+              "output": [],
+              "retries": 0,
+              "state": "passed",
+              "end": "2020-10-27T01:14:33.643Z"
+            }
+          ],
+          "hooks": [],
+          "suites": [],
+          "hooksAndTests": [],
+          "end": "2020-10-27T01:14:33.643Z"
+        }
+      ],
+      "hooksAndTests": [],
+      "end": "2020-10-27T01:14:33.644Z"
+    }
+  ],
+  "tests": []
+}

packages/terra-functional-testing/tests/jest/commands/axe/inject.test.js

@@ -15,14 +15,14 @@ describe('Inject Axe', () => {
 
   it('should inject axe into the document with the provided options', () => {
     const mockExecute = jest.fn();
-    const mockRules = { rules: [{ id: 'mock-rule', enabled: true }] };
+    const mockRules = { rules: { 'mock-rule': { enabled: true } } };
 
     global.browser = {
       execute: mockExecute,
     };
 
     injectAxe(mockRules);
 
-    expect(mockExecute).toHaveBeenCalledWith(expect.stringContaining('axe.configure({"rules":[{"id":"mock-rule","enabled":true}]})'));
+    expect(mockExecute).toHaveBeenCalledWith(expect.stringContaining('axe.configure({"rules":[{"enabled":true,"id":"mock-rule"}]})'));
   });
 });