core(duplicated-javascript): new audit

lighthouse-core/audits/byte-efficiency/bundle-duplication.js

@@ -0,0 +1,139 @@
+/**
+ * @license Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const ByteEfficiencyAudit = require('./byte-efficiency-audit.js');
+const JavascriptDuplication = require('../../computed/javascript-duplication.js');
+const i18n = require('../../lib/i18n/i18n.js');
+
+// TODO: write these.
+const UIStrings = {
+  /** Imperative title of a Lighthouse audit that tells the user to remove content from their CSS that isn’t needed immediately and instead load that content at a later time. This is displayed in a list of audit titles that Lighthouse generates. */
+  title: 'Remove duplicated modules in JavaScript bundles',
+  /** Description of a Lighthouse audit that tells the user *why* they should defer loading any content in CSS that isn’t needed at page load. This is displayed after a user expands the section to see more. No word length limits. 'Learn More' becomes link text to additional documentation. */
+  description: 'Remove dead rules from stylesheets and defer the loading of CSS not used for ' +
+    'above-the-fold content to reduce unnecessary bytes consumed by network activity. ' +
+    '[Learn more](https://web.dev/unused-css-rules).',
+};
+
+const str_ = i18n.createMessageInstanceIdFn(__filename, UIStrings);
+
+const IGNORE_THRESHOLD_IN_BYTES = 1024;
+
+class BundleDuplication extends ByteEfficiencyAudit {
+  /**
+   * @return {LH.Audit.Meta}
+   */
+  static get meta() {
+    return {
+      id: 'bundle-duplication',
+      title: str_(UIStrings.title),
+      description: str_(UIStrings.description),
+      scoreDisplayMode: ByteEfficiencyAudit.SCORING_MODES.NUMERIC,
+      requiredArtifacts: ['devtoolsLogs', 'traces', 'SourceMaps', 'ScriptElements'],
+    };
+  }
+
+  /**
+   * @param {LH.Artifacts} artifacts
+   * @param {Array<LH.Artifacts.NetworkRequest>} networkRecords
+   * @param {LH.Audit.Context} context
+   * @return {Promise<ByteEfficiencyAudit.ByteEfficiencyProduct>}
+   */
+  static async audit_(artifacts, networkRecords, context) {
+    const sourceDataAggregated = await JavascriptDuplication.request(artifacts, context);
+
+    /**
+     * @typedef ItemSubrows
+     * @property {string[]} urls
+     * @property {number[]} sourceBytes
+     */
+
+    /**
+     * @typedef {LH.Audit.ByteEfficiencyItem & ItemSubrows} Item
+     */
+
+    /** @type {Item[]} */
+    const items = [];
+
+    /** @type {Map<string, number>} */
+    const wastedBytesByUrl = new Map();
+    for (const [source, sourceDatas] of sourceDataAggregated.entries()) {
+      // One copy of this module is treated as the canonical version - the rest will have
+      // non-zero `wastedBytes`. In the case of all copies being the same version, all sizes are
+      // equal and the selection doesn't matter. When the copies are different versions, it does
+      // matter. Ideally the newest version would be the canonical copy, but version information
+      // is not present. Instead, size is used as a heuristic for latest version. This makes the
+      // audit conserative in its estimation.
+      // TODO: instead, choose the "first" script in the DOM as the canonical?
+
+      sourceDatas.sort((a, b) => b.size - a.size);
+      const urls = [];
+      const bytesValues = [];
+      let wastedBytesTotal = 0;
+      for (let i = 0; i < sourceDatas.length; i++) {
+        const sourceData = sourceDatas[i];
+        const url = sourceData.scriptUrl;
+        urls.push(url);
+        bytesValues.push(sourceData.size);
+        if (i === 0) continue;
+        wastedBytesTotal += sourceData.size;
+        wastedBytesByUrl.set(url, (wastedBytesByUrl.get(url) || 0) + sourceData.size);
+      }
+
+      items.push({
+        source,
+        wastedBytes: wastedBytesTotal,
+        // Not needed, but keeps typescript happy.
+        url: '',
+        // Not needed, but keeps typescript happy.
+        totalBytes: 0,
+        urls,
+        sourceBytes: bytesValues,
+      });
+    }
+
+    /** @type {Item} */
+    const otherItem = {
+      source: 'Other',
+      wastedBytes: 0,
+      url: '',
+      totalBytes: 0,
+      urls: [],
+      sourceBytes: [],
+    };
+    for (const item of items.filter(item => item.wastedBytes <= IGNORE_THRESHOLD_IN_BYTES)) {
+      otherItem.wastedBytes += item.wastedBytes;
+      for (let i = 0; i < item.urls.length; i++) {
+        const url = item.urls[i];
+        if (!otherItem.urls.includes(url)) {
+          otherItem.urls.push(url);
+        }
+      }
+      items.splice(items.indexOf(item), 1);
+    }
+    if (otherItem.wastedBytes) {
+      items.push(otherItem);
+    }
+
+    /** @type {LH.Audit.Details.OpportunityColumnHeading[]} */
+    const headings = [
+      {key: 'source', valueType: 'code', subRows: {key: 'urls', valueType: 'url'}, label: str_(i18n.UIStrings.columnName)}, // TODO: or 'Source'?
+      {key: '_', valueType: 'bytes', subRows: {key: 'sourceBytes'}, granularity: 0.05, label: str_(i18n.UIStrings.columnSize)},
+      {key: 'wastedBytes', valueType: 'bytes', granularity: 0.05, label: str_(i18n.UIStrings.columnWastedBytes)},
+    ];
+
+    // TODO: show warning somewhere if no source maps.
+    return {
+      items,
+      headings,
+      wastedBytesByUrl,
+    };
+  }
+}
+
+module.exports = BundleDuplication;
+module.exports.UIStrings = UIStrings;

lighthouse-core/audits/byte-efficiency/byte-efficiency-audit.js

@@ -27,6 +27,7 @@ const WASTED_MS_FOR_SCORE_OF_ZERO = 5000;
 /**
  * @typedef {object} ByteEfficiencyProduct
  * @property {Array<LH.Audit.ByteEfficiencyItem>} items
+ * @property {Map<string, number>=} wastedBytesByUrl
  * @property {LH.Audit.Details.Opportunity['headings']} headings
  * @property {string} [displayValue]
  * @property {string} [explanation]
@@ -65,7 +66,7 @@ class UnusedBytes extends Audit {
    *
    * @param {LH.Artifacts.NetworkRequest=} networkRecord
    * @param {number} totalBytes Uncompressed size of the resource
-   * @param {LH.Crdp.Network.ResourceType=} resourceType
+   * @param {LH.Crdp.Page.ResourceType=} resourceType
    * @return {number}
    */
   static estimateTransferSize(networkRecord, totalBytes, resourceType) {
@@ -135,7 +136,7 @@ class UnusedBytes extends Audit {
    * @param {Array<LH.Audit.ByteEfficiencyItem>} results The array of byte savings results per resource
    * @param {Node} graph
    * @param {Simulator} simulator
-   * @param {{includeLoad?: boolean, label?: string}=} options
+   * @param {{includeLoad?: boolean, label?: string, providedWastedBytesByUrl?: Map<string, number>}=} options
    * @return {number}
    */
   static computeWasteWithTTIGraph(results, graph, simulator, options) {
@@ -144,24 +145,26 @@ class UnusedBytes extends Audit {
     const afterLabel = `${options.label}-after`;
 
     const simulationBeforeChanges = simulator.simulate(graph, {label: beforeLabel});
-    /** @type {Map<string, LH.Audit.ByteEfficiencyItem>} */
-    const resultsByUrl = new Map();
-    for (const result of results) {
-      resultsByUrl.set(result.url, result);
+
+    let wastedBytesByUrl = options.providedWastedBytesByUrl || new Map();
+    if (!wastedBytesByUrl.size) {
+      wastedBytesByUrl = new Map();
+      for (const {url, wastedBytes} of results) {
+        wastedBytesByUrl.set(url, (wastedBytesByUrl.get(url) || 0) + wastedBytes);
+      }
     }
 
     // Update all the transfer sizes to reflect implementing our recommendations
     /** @type {Map<string, number>} */
     const originalTransferSizes = new Map();
     graph.traverse(node => {
       if (node.type !== 'network') return;
-      const result = resultsByUrl.get(node.record.url);
-      if (!result) return;
+      const wastedBytes = wastedBytesByUrl.get(node.record.url);
+      if (!wastedBytes) return;
 
       const original = node.record.transferSize;
       originalTransferSizes.set(node.record.requestId, original);
 
-      const wastedBytes = result.wastedBytes;
       node.record.transferSize = Math.max(original - wastedBytes, 0);
     });
 
@@ -197,7 +200,9 @@ class UnusedBytes extends Audit {
 
     const wastedBytes = results.reduce((sum, item) => sum + item.wastedBytes, 0);
     const wastedKb = Math.round(wastedBytes / KB_IN_BYTES);
-    const wastedMs = this.computeWasteWithTTIGraph(results, graph, simulator);
+    const wastedMs = this.computeWasteWithTTIGraph(results, graph, simulator, {
+      providedWastedBytesByUrl: result.wastedBytesByUrl,
+    });
 
     let displayValue = result.displayValue || '';
     if (typeof result.displayValue === 'undefined' && wastedBytes) {
@@ -211,7 +216,6 @@ class UnusedBytes extends Audit {
       warnings: result.warnings,
       displayValue,
       numericValue: wastedMs,
-      numericUnit: 'millisecond',
       score: UnusedBytes.scoreForWastedMs(wastedMs),
       extendedInfo: {
         value: {

lighthouse-core/computed/javascript-duplication.js

@@ -0,0 +1,103 @@
+/**
+ * @license Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const makeComputedArtifact = require('./computed-artifact.js');
+const JsBundles = require('./js-bundles.js');
+
+class JavascriptDuplication {
+  /**
+   * @param {string} source
+   */
+  static _normalizeSource(source) {
+    // Trim trailing question mark - b/c webpack.
+    source = source.replace(/\?$/, '');
+
+    // Normalize paths for dependencies by only keeping everything after the last `node_modules`.
+    const lastNodeModulesIndex = source.lastIndexOf('node_modules');
+    if (lastNodeModulesIndex !== -1) {
+      source = source.substring(lastNodeModulesIndex);
+    }
+
+    return source;
+  }
+
+  /**
+   * @param {string} source
+   */
+  static _shouldIgnoreSource(source) {
+    // Ignore bundle overhead.
+    if (source.includes('webpack/bootstrap')) return true;
+    if (source.includes('(webpack)/buildin')) return true;
+
+    // Ignore shims.
+    if (source.includes('external ')) return true;
+
+    return false;
+  }
+
+  /**
+   * @param {LH.Artifacts} artifacts
+   * @param {LH.Audit.Context} context
+   */
+  static async compute_(artifacts, context) {
+    const bundles = await JsBundles.request(artifacts, context);
+
+    /**
+     * @typedef SourceData
+     * @property {string} source
+     * @property {number} size
+     */
+
+    /** @type {Map<LH.Artifacts.RawSourceMap, SourceData[]>} */
+    const sourceDatasMap = new Map();
+
+    // Determine size of each `sources` entry.
+    for (const {rawMap, sizes} of bundles) {
+      /** @type {SourceData[]} */
+      const sourceDatas = [];
+      sourceDatasMap.set(rawMap, sourceDatas);
+
+      for (let i = 0; i < rawMap.sources.length; i++) {
+        if (this._shouldIgnoreSource(rawMap.sources[i])) continue;
+
+        const fullSource = (rawMap.sourceRoot || '') + rawMap.sources[i];
+        const sourceSize = sizes.files[fullSource];
+        sourceDatas.push({
+          source: JavascriptDuplication._normalizeSource(rawMap.sources[i]),
+          size: sourceSize,
+        });
+      }
+    }
+
+    /** @type {Map<string, Array<{scriptUrl: string, size: number}>>} */
+    const sourceDataAggregated = new Map();
+    for (const {rawMap, script} of bundles) {
+      const sourceDatas = sourceDatasMap.get(rawMap);
+      if (!sourceDatas) continue;
+
+      for (const sourceData of sourceDatas) {
+        let data = sourceDataAggregated.get(sourceData.source);
+        if (!data) {
+          data = [];
+          sourceDataAggregated.set(sourceData.source, data);
+        }
+        data.push({
+          scriptUrl: script.src || '',
+          size: sourceData.size,
+        });
+      }
+    }
+
+    for (const [key, value] of sourceDataAggregated.entries()) {
+      if (value.length === 1) sourceDataAggregated.delete(key);
+    }
+
+    return sourceDataAggregated;
+  }
+}
+
+module.exports = makeComputedArtifact(JavascriptDuplication);

lighthouse-core/config/experimental-config.js

@@ -20,13 +20,15 @@ const config = {
     ],
   }],
   audits: [
+    'byte-efficiency/bundle-duplication',
     'legacy-javascript',
   ],
   // @ts-ignore: `title` is required in CategoryJson. setting to the same value as the default
   // config is awkward - easier to omit the property here. Will defer to default config.
   categories: {
     'performance': {
       auditRefs: [
+        {id: 'bundle-duplication', weight: 0, group: 'load-opportunities'},
         {id: 'legacy-javascript', weight: 0, group: 'diagnostics'},
       ],
     },