core: add top-level runtimeError

lighthouse-core/lib/lh-error.js

@@ -12,18 +12,20 @@ const strings = require('./strings');
  * @property {string} code
  * @property {string} message
  * @property {RegExp} [pattern]
+ * @property {boolean} [lhrRuntimeError] True if it should appear in the top-level LHR.runtimeError property.
  */
 
 class LighthouseError extends Error {
   /**
    * @param {LighthouseErrorDefinition} errorDefinition
-   * @param {Record<string, string|undefined>=} properties
+   * @param {Record<string, string|boolean|undefined>=} properties
    */
   constructor(errorDefinition, properties) {
     super(errorDefinition.code);
     this.name = 'LHError';
     this.code = errorDefinition.code;
     this.friendlyMessage = errorDefinition.message;
+    this.lhrRuntimeError = !!errorDefinition.lhrRuntimeError;
     if (properties) Object.assign(this, properties);
 
     Error.captureStackTrace(this, LighthouseError);
@@ -75,17 +77,52 @@ class LighthouseError extends Error {
 
 const ERRORS = {
   // Screenshot/speedline errors
-  NO_SPEEDLINE_FRAMES: {code: 'NO_SPEEDLINE_FRAMES', message: strings.didntCollectScreenshots},
-  SPEEDINDEX_OF_ZERO: {code: 'SPEEDINDEX_OF_ZERO', message: strings.didntCollectScreenshots},
-  NO_SCREENSHOTS: {code: 'NO_SCREENSHOTS', message: strings.didntCollectScreenshots},
-  INVALID_SPEEDLINE: {code: 'INVALID_SPEEDLINE', message: strings.didntCollectScreenshots},
+  NO_SPEEDLINE_FRAMES: {
+    code: 'NO_SPEEDLINE_FRAMES',
+    message: strings.didntCollectScreenshots,
+    lhrRuntimeError: true,
+  },
+  SPEEDINDEX_OF_ZERO: {
+    code: 'SPEEDINDEX_OF_ZERO',
+    message: strings.didntCollectScreenshots,
+    lhrRuntimeError: true,
+  },
+  NO_SCREENSHOTS: {
+    code: 'NO_SCREENSHOTS',
+    message: strings.didntCollectScreenshots,
+    lhrRuntimeError: true,
+  },
+  INVALID_SPEEDLINE: {
+    code: 'INVALID_SPEEDLINE',
+    message: strings.didntCollectScreenshots,
+    lhrRuntimeError: true,
+  },
 
   // Trace parsing errors
-  NO_TRACING_STARTED: {code: 'NO_TRACING_STARTED', message: strings.badTraceRecording},
-  NO_NAVSTART: {code: 'NO_NAVSTART', message: strings.badTraceRecording},
-  NO_FCP: {code: 'NO_FCP', message: strings.badTraceRecording},
-  NO_FMP: {code: 'NO_FMP', message: strings.badTraceRecording},
-  NO_DCL: {code: 'NO_DCL', message: strings.badTraceRecording},
+  NO_TRACING_STARTED: {
+    code: 'NO_TRACING_STARTED',
+    message: strings.badTraceRecording,
+    lhrRuntimeError: true,
+  },
+  NO_NAVSTART: {
+    code: 'NO_NAVSTART',
+    message: strings.badTraceRecording,
+    lhrRuntimeError: true,
+  },
+  NO_FCP: {
+    code: 'NO_FCP',
+    message: strings.badTraceRecording,
+    lhrRuntimeError: true,
+  },
+  NO_DCL: {
+    code: 'NO_DCL',
+    message: strings.badTraceRecording,
+    lhrRuntimeError: true,
+  },
+  NO_FMP: {
+    code: 'NO_FMP',
+    message: strings.badTraceRecording,
+  },
 
   // TTI calculation failures
   FMP_TOO_LATE_FOR_FCPUI: {code: 'FMP_TOO_LATE_FOR_FCPUI', message: strings.pageLoadTookTooLong},
@@ -97,21 +134,36 @@ const ERRORS = {
   },
 
   // Page load failures
-  NO_DOCUMENT_REQUEST: {code: 'NO_DOCUMENT_REQUEST', message: strings.pageLoadFailed},
-  FAILED_DOCUMENT_REQUEST: {code: 'FAILED_DOCUMENT_REQUEST', message: strings.pageLoadFailed},
+  NO_DOCUMENT_REQUEST: {
+    code: 'NO_DOCUMENT_REQUEST',
+    message: strings.pageLoadFailed,
+    lhrRuntimeError: true,
+  },
+  FAILED_DOCUMENT_REQUEST: {
+    code: 'FAILED_DOCUMENT_REQUEST',
+    message: strings.pageLoadFailed,
+    lhrRuntimeError: true,
+  },
 
   // Protocol internal failures
   TRACING_ALREADY_STARTED: {
     code: 'TRACING_ALREADY_STARTED',
     message: strings.internalChromeError,
     pattern: /Tracing.*started/,
+    lhrRuntimeError: true,
   },
   PARSING_PROBLEM: {
     code: 'PARSING_PROBLEM',
     message: strings.internalChromeError,
     pattern: /Parsing problem/,
+    lhrRuntimeError: true,
+  },
+  READ_FAILED: {
+    code: 'READ_FAILED',
+    message: strings.internalChromeError,
+    pattern: /Read failed/,
+    lhrRuntimeError: true,
   },
-  READ_FAILED: {code: 'READ_FAILED', message: strings.internalChromeError, pattern: /Read failed/},
 
   // Protocol timeout failures
   REQUEST_CONTENT_TIMEOUT: {
@@ -122,5 +174,6 @@ const ERRORS = {
 
 /** @type {Record<keyof typeof ERRORS, LighthouseErrorDefinition>} */
 LighthouseError.errors = ERRORS;
+LighthouseError.NO_ERROR = 'NO_ERROR';
 module.exports = LighthouseError;
 

lighthouse-core/runner.js

@@ -18,6 +18,7 @@ const path = require('path');
 const URL = require('./lib/url-shim');
 const Sentry = require('./lib/sentry');
 const generateReport = require('./report/report-generator').generateReport;
+const LHError = require('./lib/lh-error.js');
 
 /** @typedef {import('./gather/connections/connection.js')} Connection */
 /** @typedef {import('./config/config.js')} Config */
@@ -131,6 +132,7 @@ class Runner {
         requestedUrl: requestedUrl,
         finalUrl: artifacts.URL.finalUrl,
         runWarnings: lighthouseRunWarnings,
+        runtimeError: Runner.getArtifactRuntimeError(artifacts),
         audits: resultsById,
         configSettings: settings,
         categories,
@@ -291,6 +293,31 @@ class Runner {
     return auditResult;
   }
 
+  /**
+   * Returns first runtimeError found in artifacts.
+   * @param {LH.Artifacts} artifacts
+   * @return {LH.Result['runtimeError']}
+   */
+  static getArtifactRuntimeError(artifacts) {
+    for (const possibleErrorArtifact of Object.values(artifacts)) {
+      if (possibleErrorArtifact instanceof LHError && possibleErrorArtifact.lhrRuntimeError) {
+        const errorMessage = possibleErrorArtifact.friendlyMessage ?
+            `${possibleErrorArtifact.friendlyMessage} (${possibleErrorArtifact.message})` :
+            possibleErrorArtifact.message;
+
+        return {
+          code: possibleErrorArtifact.code,
+          message: errorMessage,
+        };
+      }
+    }
+
+    return {
+      code: LHError.NO_ERROR,
+      message: '',
+    };
+  }
+
   /**
    * Returns list of audit names for external querying.
    * @return {Array<string>}

lighthouse-core/test/results/sample_v2.json

@@ -10,6 +10,10 @@
   "requestedUrl": "http://localhost:10200/dobetterweb/dbw_tester.html",
   "finalUrl": "http://localhost:10200/dobetterweb/dbw_tester.html",
   "runWarnings": [],
+  "runtimeError": {
+    "code": "NO_ERROR",
+    "message": ""
+  },
   "audits": {
     "is-on-https": {
       "id": "is-on-https",

lighthouse-core/test/runner-test.js

@@ -10,12 +10,14 @@ const GatherRunner = require('../gather/gather-runner');
 const driverMock = require('./gather/fake-driver');
 const Config = require('../config/config');
 const Audit = require('../audits/audit');
+const Gatherer = require('../gather/gatherers/gatherer.js');
 const assetSaver = require('../lib/asset-saver');
 const fs = require('fs');
 const assert = require('assert');
 const path = require('path');
 const sinon = require('sinon');
 const rimraf = require('rimraf');
+const LHError = require('../lib/lh-error.js');
 
 /* eslint-env jest */
 
@@ -605,6 +607,41 @@ describe('Runner', () => {
     });
   });
 
+  it('includes a top-level runtimeError when a gatherer throws one', async () => {
+    const NO_FCP = LHError.errors.NO_FCP;
+    class RuntimeErrorGatherer extends Gatherer {
+      afterPass() {
+        throw new LHError(NO_FCP);
+      }
+    }
+    class WarningAudit extends Audit {
+      static get meta() {
+        return {
+          id: 'test-audit',
+          title: 'A test audit',
+          description: 'An audit for testing',
+          requiredArtifacts: ['RuntimeErrorGatherer'],
+        };
+      }
+      static audit() {
+        throw new Error('Should not get here');
+      }
+    }
+
+    const config = new Config({
+      passes: [{gatherers: [RuntimeErrorGatherer]}],
+      audits: [WarningAudit],
+    });
+    const {lhr} = await Runner.run(null, {url: 'https://example.com/', config, driverMock});
+
+    // Audit error included the runtimeError
+    assert.strictEqual(lhr.audits['test-audit'].scoreDisplayMode, 'error');
+    assert.ok(lhr.audits['test-audit'].errorMessage.includes(NO_FCP.code));
+    // And it bubbled up to the runtimeError.
+    assert.strictEqual(lhr.runtimeError.code, NO_FCP.code);
+    assert.ok(lhr.runtimeError.message.includes(NO_FCP.code));
+  });
+
   it('can handle array of outputs', async () => {
     const url = 'https://example.com';
     const config = new Config({

lighthouse-extension/app/src/lighthouse-ext-background.js

@@ -11,6 +11,7 @@ const background = require('./lighthouse-background');
 const ExtensionProtocol = require('../../../lighthouse-core/gather/connections/extension');
 const log = require('lighthouse-logger');
 const assetSaver = require('../../../lighthouse-core/lib/asset-saver.js');
+const LHError = require('../../../lighthouse-core/lib/lh-error.js');
 
 /** @type {Record<'mobile'|'desktop', LH.Config.Json>} */
 const LR_PRESETS = {
@@ -111,14 +112,29 @@ async function runLighthouseInLR(connection, url, flags, {lrDevice, categoryIDs,
     config.settings.onlyCategories = categoryIDs;
   }
 
-  const results = await lighthouse(url, flags, config, connection);
-  if (!results) return;
+  try {
+    const results = await lighthouse(url, flags, config, connection);
+    if (!results) return;
 
-  if (logAssets) {
-    await assetSaver.logAssets(results.artifacts, results.lhr.audits);
-  }
+    if (logAssets) {
+      await assetSaver.logAssets(results.artifacts, results.lhr.audits);
+    }
+    return results.report;
+  } catch (err) {
+    // If an error ruined the entire lighthouse run, attempt to return a meaningful error.
+    if (!(err instanceof LHError) || !err.lhrRuntimeError) {
+      throw err;
+    }
 
-  return results.report;
+    const runtimeError = {
+      code: err.code,
+      message: err.friendlyMessage ?
+          `${err.friendlyMessage} (${err.message})` :
+          err.message,
+    };
+
+    return JSON.stringify({runtimeError}, null, 2);
+  }
 }
 
 /**
@@ -222,28 +238,30 @@ if ('chrome' in window && chrome.runtime) {
 }
 
 if (typeof module !== 'undefined' && module.exports) {
-  // Export for popup.js to import types. We don't want tsc to infer an index
-  // type, so use exports instead of module.exports.
-  exports.runLighthouseInExtension = runLighthouseInExtension;
-  exports.getDefaultCategories = background.getDefaultCategories;
-  exports.isRunning = isRunning;
-  exports.listenForStatus = listenForStatus;
-  exports.saveSettings = saveSettings;
-  exports.loadSettings = loadSettings;
+  // Export for popup.js to import types.
+  module.exports = {
+    runLighthouseInExtension,
+    runLighthouseInLR,
+    getDefaultCategories: background.getDefaultCategories,
+    isRunning,
+    listenForStatus,
+    saveSettings,
+    loadSettings,
+  };
+} else {
+  // Expose on window for extension, other consumers of file.
+  // @ts-ignore
+  window.runLighthouseInExtension = runLighthouseInExtension;
+  // @ts-ignore
+  window.runLighthouseInLR = runLighthouseInLR;
+  // @ts-ignore
+  window.getDefaultCategories = background.getDefaultCategories;
+  // @ts-ignore
+  window.isRunning = isRunning;
+  // @ts-ignore
+  window.listenForStatus = listenForStatus;
+  // @ts-ignore
+  window.loadSettings = loadSettings;
+  // @ts-ignore
+  window.saveSettings = saveSettings;
 }
-
-// Expose on window for extension, other consumers of file.
-// @ts-ignore
-window.runLighthouseInExtension = runLighthouseInExtension;
-// @ts-ignore
-window.runLighthouseInLR = runLighthouseInLR;
-// @ts-ignore
-window.getDefaultCategories = background.getDefaultCategories;
-// @ts-ignore
-window.isRunning = isRunning;
-// @ts-ignore
-window.listenForStatus = listenForStatus;
-// @ts-ignore
-window.loadSettings = loadSettings;
-// @ts-ignore
-window.saveSettings = saveSettings;

lighthouse-extension/test/app/src/lighthouse-ext-background-test.js

@@ -0,0 +1,33 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const assert = require('assert');
+const lhBackground = require('../../../app/src/lighthouse-ext-background.js');
+const LHError = require('../../../../lighthouse-core/lib/lh-error.js');
+
+/* eslint-env mocha */
+
+describe('lighthouse-ext-background', () => {
+  it('returns a runtimeError LHR when lighthouse run throws a runtimeError', async () => {
+    const connectionError = new LHError(LHError.errors.FAILED_DOCUMENT_REQUEST);
+    assert.strictEqual(connectionError.lhrRuntimeError, true);
+    const mockConnection = {
+      async connect() {
+        throw connectionError;
+      },
+      async disconnect() {},
+      async sendCommand() {},
+      on() {},
+    };
+    const url = 'https://example.com';
+
+    const result = await lhBackground.runLighthouseInLR(mockConnection, url, {output: 'json'}, {});
+    const parsedResult = JSON.parse(result);
+    assert.strictEqual(parsedResult.runtimeError.code, connectionError.code);
+    assert.ok(parsedResult.runtimeError.message.includes(connectionError.friendlyMessage));
+  });
+});

typings/lhr.d.ts

@@ -4,6 +4,8 @@
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  */
 
+import LHError = require('../lighthouse-core/lib/lh-error.js');
+
 declare global {
   module LH {
     export type I18NMessageEntry = string | {path: string, values: any};
@@ -50,6 +52,8 @@ declare global {
       configSettings: Config.Settings;
       /** List of top-level warnings for this Lighthouse run. */
       runWarnings: string[];
+      /** A top-level error message that, if present, indicates a serious enough problem that this Lighthouse result may need to be discarded. */
+      runtimeError: {code: string, message: string};
       /** The User-Agent string of the browser used run Lighthouse for these results. */
       userAgent: string;
       /** Information about the environment in which Lighthouse was run. */