core(audits): add handling of 'incomplete' results from axe-core

lighthouse-cli/test/smokehouse/test-definitions/a11y/expectations.js

@@ -313,7 +313,8 @@ const expectations = [
           },
         },
         'form-field-multiple-labels': {
-          score: 0,
+          score: null,
+          scoreDisplayMode: 'informative',
           details: {
             items: [
               {
@@ -322,7 +323,6 @@ const expectations = [
                   'selector': '#form-field-multiple-labels',
                   'path': '2,HTML,1,BODY,35,SECTION,2,INPUT',
                   'snippet': '<input type="checkbox" id="form-field-multiple-labels">',
-                  'explanation': 'Fix all of the following:\n  Multiple label elements is not widely supported in assistive technologies',
                   'nodeLabel': 'input',
                 },
               },

lighthouse-core/audits/accessibility/axe-audit.js

@@ -22,6 +22,9 @@ const str_ = i18n.createMessageInstanceIdFn(__filename, UIStrings);
 
 class AxeAudit extends Audit {
   /**
+   * Base class for audit rules which reflect assessment performed by the aXe accessibility library
+   * See https://github.com/dequelabs/axe-core/blob/6b444546cff492a62a70a74a8fc3c62bd4729400/doc/API.md#results-object for result type and format details
+   *
    * @param {LH.Artifacts} artifacts Accessibility gatherer artifacts. Note that AxeAudit
    * expects the meta name for the class to match the rule id from aXe.
    * @return {LH.Audit.Product}
@@ -39,11 +42,32 @@ class AxeAudit extends Audit {
       };
     }
 
+    // Detect errors reported within aXe 'incomplete' results
+    // aXe uses this result type to indicate errors, or rules which require manual investigation
+    // If aXe reports an error, then bubble it up to the caller
+    const incomplete = artifacts.Accessibility.incomplete || [];
+    const incompleteResult = incomplete.find(result => result.id === this.meta.id);
+    if (incompleteResult && incompleteResult.error) {
+      return Audit.generateErrorAuditResult(this, incompleteResult.error.message);
+    }
+
+    const isInformative = this.meta.scoreDisplayMode === Audit.SCORING_MODES.INFORMATIVE;
     const violations = artifacts.Accessibility.violations || [];
-    const rule = violations.find(result => result.id === this.meta.id);
+    const failureCases = isInformative ? violations.concat(incomplete) : violations;
+    const rule = failureCases.find(result => result.id === this.meta.id);
     const impact = rule && rule.impact;
     const tags = rule && rule.tags;
 
+    // Handle absence of aXe failure results for informative rules as 'not applicable'
+    // This scenario indicates that no action is required by the web property owner
+    // Since there is no score impact from informative rules, display the rule as not applicable
+    if (isInformative && !rule) {
+      return {
+        score: 1,
+        notApplicable: true,
+      };
+    }
+
     /** @type {LH.Audit.Details.Table['items']}>} */
     let items = [];
     if (rule && rule.nodes) {

lighthouse-core/audits/accessibility/form-field-multiple-labels.js

@@ -34,6 +34,7 @@ class FormFieldMultipleLabels extends AxeAudit {
       title: str_(UIStrings.title),
       failureTitle: str_(UIStrings.failureTitle),
       description: str_(UIStrings.description),
+      scoreDisplayMode: AxeAudit.SCORING_MODES.INFORMATIVE,
       requiredArtifacts: ['Accessibility'],
     };
   }

lighthouse-core/gather/gatherers/accessibility.js

@@ -47,21 +47,31 @@ function runA11yChecks() {
     },
     // @ts-ignore
   }).then(axeResult => {
-    // Augment the node objects with outerHTML snippet & custom path string
     // @ts-ignore
-    axeResult.violations.forEach(v => v.nodes.forEach(node => {
-      // @ts-ignore - getNodePath put into scope via stringification
-      node.path = getNodePath(node.element);
-      // @ts-ignore - getOuterHTMLSnippet put into scope via stringification
-      node.snippet = getOuterHTMLSnippet(node.element);
-      // @ts-ignore - getNodeLabel put into scope via stringification
-      node.nodeLabel = getNodeLabel(node.element);
-      // avoid circular JSON concerns
-      node.element = node.any = node.all = node.none = undefined;
-    }));
+    const augmentAxeNodes = result => {
+      // @ts-ignore
+      result.nodes.forEach(node => {
+        // @ts-ignore - getNodePath put into scope via stringification
+        node.path = getNodePath(node.element);
+        // @ts-ignore - getOuterHTMLSnippet put into scope via stringification
+        node.snippet = getOuterHTMLSnippet(node.element);
+        // @ts-ignore - getNodeLabel put into scope via stringification
+        node.nodeLabel = getNodeLabel(node.element);
+        // avoid circular JSON concerns
+        node.element = node.any = node.all = node.none = undefined;
+      });
+    };
+
+    // Augment the node objects with outerHTML snippet & custom path string
+    axeResult.violations.forEach(augmentAxeNodes);
+    axeResult.incomplete.forEach(augmentAxeNodes);
 
     // We only need violations, and circular references are possible outside of violations
-    axeResult = {violations: axeResult.violations, notApplicable: axeResult.inapplicable};
+    axeResult = {
+      violations: axeResult.violations,
+      notApplicable: axeResult.inapplicable,
+      incomplete: axeResult.incomplete,
+    };
     return axeResult;
   });
 }

lighthouse-core/test/audits/accessibility/axe-audit-test.js

@@ -35,5 +35,113 @@ describe('Accessibility: axe-audit', () => {
       const output = FakeA11yAudit.audit(artifacts);
       assert.equal(output.score, 0);
     });
+
+    it('returns axe error message to the caller when present', () => {
+      class FakeA11yAudit extends AxeAudit {
+        static get meta() {
+          return {
+            id: 'fake-incomplete-error',
+            title: 'Example title',
+            requiredArtifacts: ['Accessibility'],
+          };
+        }
+      }
+      const artifacts = {
+        Accessibility: {
+          incomplete: [{
+            id: 'fake-incomplete-error',
+            nodes: [],
+            help: 'http://example.com/',
+            error: {
+              name: 'SupportError',
+              message: 'Feature is not supported on your platform',
+            },
+          }],
+        },
+      };
+
+      const output = FakeA11yAudit.audit(artifacts);
+      assert.equal(output.errorMessage, 'Feature is not supported on your platform');
+    });
+
+    it('considers passing axe result as not applicable for informative audit', () => {
+      class FakeA11yAudit extends AxeAudit {
+        static get meta() {
+          return {
+            id: 'fake-axe-pass',
+            title: 'Example title',
+            scoreDisplayMode: 'informative',
+            requiredArtifacts: ['Accessibility'],
+          };
+        }
+      }
+      const artifacts = {
+        Accessibility: {
+          passes: [{
+            id: 'fake-axe-pass',
+            help: 'http://example.com/',
+          }],
+        },
+      };
+
+      const output = FakeA11yAudit.audit(artifacts);
+      assert.ok(output.notApplicable);
+    });
+
+    it('considers failing axe result as failure for informative audit', () => {
+      class FakeA11yAudit extends AxeAudit {
+        static get meta() {
+          return {
+            id: 'fake-axe-failure-case',
+            title: 'Example title',
+            scoreDisplayMode: 'informative',
+            requiredArtifacts: ['Accessibility'],
+          };
+        }
+      }
+      const artifacts = {
+        Accessibility: {
+          incomplete: [{
+            id: 'fake-axe-failure-case',
+            nodes: [{html: '<input id="multi-label-form-element" />'}],
+            help: 'http://example.com/',
+          }],
+          // TODO: remove: axe-core v3.3.0 backwards-compatibility test
+          violations: [{
+            id: 'fake-axe-failure-case',
+            nodes: [{html: '<input id="multi-label-form-element" />'}],
+            help: 'http://example.com/',
+          }],
+        },
+      };
+
+      const output = FakeA11yAudit.audit(artifacts);
+      assert.ok(!output.notApplicable);
+      assert.equal(output.score, 0);
+    });
+
+    it('considers error-free incomplete axe result as failure for informative audit', () => {
+      class FakeA11yAudit extends AxeAudit {
+        static get meta() {
+          return {
+            id: 'fake-incomplete-fail',
+            title: 'Example title',
+            scoreDisplayMode: 'informative',
+            requiredArtifacts: ['Accessibility'],
+          };
+        }
+      }
+      const artifacts = {
+        Accessibility: {
+          incomplete: [{
+            id: 'fake-incomplete-fail',
+            help: 'http://example.com/',
+          }],
+        },
+      };
+
+      const output = FakeA11yAudit.audit(artifacts);
+      assert.equal(output.score, 0);
+    });
   });
 });

lighthouse-core/test/report/html/renderer/category-renderer-test.js

@@ -242,7 +242,7 @@ describe('CategoryRenderer', () => {
       const categoryDOM = renderer.render(category, sampleResults.categoryGroups);
 
       const gauge = categoryDOM.querySelector('.lh-gauge__percentage');
-      assert.equal(gauge.textContent.trim(), '66', 'score is 0-100');
+      assert.equal(gauge.textContent.trim(), '65', 'score is 0-100');
 
       const score = categoryDOM.querySelector('.lh-category-header');
       const value = categoryDOM.querySelector('.lh-gauge__percentage');

lighthouse-core/test/results/sample_v2.json

@@ -1801,13 +1801,8 @@
       "id": "form-field-multiple-labels",
       "title": "No form fields have multiple labels",
       "description": "Form fields with multiple labels can be confusingly announced by assistive technologies like screen readers which use either the first, the last, or all of the labels. [Learn more](https://web.dev/form-field-multiple-labels/).",
-      "score": 1,
-      "scoreDisplayMode": "binary",
-      "details": {
-        "type": "table",
-        "headings": [],
-        "items": []
-      }
+      "score": null,
+      "scoreDisplayMode": "notApplicable"
     },
     "frame-title": {
       "id": "frame-title",
@@ -3819,7 +3814,7 @@
         },
         {
           "id": "form-field-multiple-labels",
-          "weight": 2,
+          "weight": 0,
           "group": "a11y-names-labels"
         },
         {
@@ -3964,7 +3959,7 @@
         }
       ],
       "id": "accessibility",
-      "score": 0.66
+      "score": 0.65
     },
     "best-practices": {
       "title": "Best Practices",

proto/sample_v2_round_trip.json

@@ -860,14 +860,9 @@
         },
         "form-field-multiple-labels": {
             "description": "Form fields with multiple labels can be confusingly announced by assistive technologies like screen readers which use either the first, the last, or all of the labels. [Learn more](https://web.dev/form-field-multiple-labels/).",
-            "details": {
-                "headings": [],
-                "items": [],
-                "type": "table"
-            },
             "id": "form-field-multiple-labels",
-            "score": 1.0,
-            "scoreDisplayMode": "binary",
+            "score": null,
+            "scoreDisplayMode": "notApplicable",
             "title": "No form fields have multiple labels"
         },
         "frame-title": {
@@ -3443,7 +3438,7 @@
                 {
                     "group": "a11y-names-labels",
                     "id": "form-field-multiple-labels",
-                    "weight": 2.0
+                    "weight": 0.0
                 },
                 {
                     "group": "a11y-names-labels",
@@ -3589,7 +3584,7 @@
             "description": "These checks highlight opportunities to [improve the accessibility of your web app](https://developers.google.com/web/fundamentals/accessibility). Only a subset of accessibility issues can be automatically detected so manual testing is also encouraged.",
             "id": "accessibility",
             "manualDescription": "These items address areas which an automated testing tool cannot cover. Learn more in our guide on [conducting an accessibility review](https://developers.google.com/web/fundamentals/accessibility/how-to-review).",
-            "score": 0.66,
+            "score": 0.65,
             "title": "Accessibility"
         },
         "best-practices": {

types/artifacts.d.ts

@@ -144,23 +144,30 @@ declare global {
       export type TaskNode = _TaskNode;
       export type MetaElement = LH.Artifacts['MetaElements'][0];
 
+      export interface RuleExecutionError {
+        name: string;
+        message: string;
+      }
+
+      export interface AxeResult {
+        id: string;
+        impact: string;
+        tags: Array<string>;
+        nodes: Array<{
+          path: string;
+          html: string;
+          snippet: string;
+          target: Array<string>;
+          failureSummary?: string;
+          nodeLabel?: string;
+        }>;
+        error?: RuleExecutionError;
+      }
+
       export interface Accessibility {
-        violations: {
-          id: string;
-          impact: string;
-          tags: string[];
-          nodes: {
-            path: string;
-            html: string;
-            snippet: string;
-            target: string[];
-            failureSummary?: string;
-            nodeLabel?: string;
-          }[];
-        }[];
-        notApplicable: {
-          id: string
-        }[];
+        violations: Array<AxeResult>;
+        notApplicable: Array<Pick<AxeResult, 'id'>>;
+        incomplete: Array<AxeResult>;
       }
 
       export interface CSSStyleSheetInfo {