From 209443b05c95bab433a93dbcabd5b43212c29ebb Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Sat, 8 Feb 2020 20:56:44 -0800 Subject: [PATCH 01/21] bundledup --- .../byte-efficiency/bundle-duplication.js | 261 ++++++++++++++++++ .../byte-efficiency/byte-efficiency-audit.js | 42 ++- lighthouse-core/config/source-maps-config.js | 31 +++ 3 files changed, 323 insertions(+), 11 deletions(-) create mode 100644 lighthouse-core/audits/byte-efficiency/bundle-duplication.js create mode 100644 lighthouse-core/config/source-maps-config.js diff --git a/lighthouse-core/audits/byte-efficiency/bundle-duplication.js b/lighthouse-core/audits/byte-efficiency/bundle-duplication.js new file mode 100644 index 000000000000..af529a821c00 --- /dev/null +++ b/lighthouse-core/audits/byte-efficiency/bundle-duplication.js @@ -0,0 +1,261 @@ +/** + * @license Copyright 2017 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const ByteEfficiencyAudit = require('./byte-efficiency-audit.js'); +const JsBundles = require('../../computed/js-bundles.js'); +const i18n = require('../../lib/i18n/i18n.js'); + +// TODO: write these. +const UIStrings = { + /** Imperative title of a Lighthouse audit that tells the user to remove content from their CSS that isn’t needed immediately and instead load that content at a later time. This is displayed in a list of audit titles that Lighthouse generates. */ + title: 'Remove duplicated code within bundles', + /** Description of a Lighthouse audit that tells the user *why* they should defer loading any content in CSS that isn’t needed at page load. This is displayed after a user expands the section to see more. No word length limits. 'Learn More' becomes link text to additional documentation. */ + description: 'Remove dead rules from stylesheets and defer the loading of CSS not used for ' + + 'above-the-fold content to reduce unnecessary bytes consumed by network activity. ' + + '[Learn more](https://web.dev/unused-css-rules).', +}; + +const str_ = i18n.createMessageInstanceIdFn(__filename, UIStrings); + +const IGNORE_THRESHOLD_IN_BYTES = 1024; + +class BundleDuplication extends ByteEfficiencyAudit { + /** + * @return {LH.Audit.Meta} + */ + static get meta() { + return { + id: 'bundle-duplication', + title: str_(UIStrings.title), + description: str_(UIStrings.description), + scoreDisplayMode: ByteEfficiencyAudit.SCORING_MODES.NUMERIC, + requiredArtifacts: ['devtoolsLogs', 'traces', 'SourceMaps', 'ScriptElements'], + }; + } + + /** + * @param {LH.Artifacts} artifacts + * @param {Array} networkRecords + * @param {LH.Audit.Context} context + * @return {Promise} + */ + static async audit_(artifacts, networkRecords, context) { + const bundles = await JsBundles.request(artifacts, context); + + /** + * @typedef SourceData + * @property {string} normalizedSource + * @property {number} size + */ + + /** @type {Map} */ + const sourceDatasMap = new Map(); + + // Determine size of each `sources` entry. + for (const {rawMap, sizes} of bundles) { + /** @type {SourceData[]} */ + const sourceDatas = []; + sourceDatasMap.set(rawMap, sourceDatas); + + for (let i = 0; i < rawMap.sources.length; i++) { + const source = rawMap.sources[i]; + // Trim trailing question mark - b/c webpack. + let normalizedSource = source.replace(/\?$/, ''); + // Normalize paths for dependencies by keeping everything after the last `node_modules`. + const lastNodeModulesIndex = normalizedSource.lastIndexOf('node_modules'); + if (lastNodeModulesIndex !== -1) { + normalizedSource = source.substring(lastNodeModulesIndex); + } + + // Ignore bundle overhead. + if (normalizedSource.includes('webpack/bootstrap')) continue; + if (normalizedSource.includes('(webpack)/buildin')) continue; + // Ignore shims. + if (normalizedSource.includes('external ')) continue; + + const fullSource = (rawMap.sourceRoot || '') + source; + const sourceSize = sizes.files[fullSource]; + + sourceDatas.push({ + normalizedSource, + size: sourceSize, + }); + } + } + + /** @type {Map>} */ + const sourceDataAggregated = new Map(); + for (const {rawMap, script} of bundles) { + const sourceDatas = sourceDatasMap.get(rawMap); + if (!sourceDatas) continue; + + for (const sourceData of sourceDatas) { + let data = sourceDataAggregated.get(sourceData.normalizedSource); + if (!data) { + data = []; + sourceDataAggregated.set(sourceData.normalizedSource, data); + } + data.push({ + scriptUrl: script.src || '', + size: sourceData.size, + }); + } + } + + /** + * @typedef ItemMulti + * @property {string[]} urls + * @property {number[]} sourceBytes + */ + + /** + * @typedef {LH.Audit.ByteEfficiencyItem & ItemMulti} Item + */ + + /** @type {Item[]} */ + const items = []; + + /** @type {Map} */ + const wastedBytesByUrl = new Map(); + for (const [key, sourceDatas] of sourceDataAggregated.entries()) { + if (sourceDatas.length === 1) continue; + + // One copy of this module is treated as the canonical version - the rest will have + // non-zero `wastedBytes`. In the case of all copies being the same version, all sizes are + // equal and the selection doesn't matter. When the copies are different versions, it does + // matter. Ideally the newest version would be the canonical copy, but version information + // is not present. Instead, size is used as a heuristic for latest version. This makes the + // audit conserative in its estimation. + // TODO: instead, choose the "first" script in the DOM as the canonical? + + sourceDatas.sort((a, b) => b.size - a.size); + const urls = []; + const bytesValues = []; + let wastedBytesTotal = 0; + for (let i = 0; i < sourceDatas.length; i++) { + const sourceData = sourceDatas[i]; + const url = sourceData.scriptUrl; + urls.push(url); + bytesValues.push(sourceData.size); + if (i === 0) continue; + wastedBytesTotal += sourceData.size; + wastedBytesByUrl.set(url, (wastedBytesByUrl.get(url) || 0) + sourceData.size); + } + + items.push({ + source: key, + wastedBytes: wastedBytesTotal, + // Not needed, but keeps typescript happy. + url: '', + // Not needed, but keeps typescript happy. + totalBytes: 0, + urls, + sourceBytes: bytesValues, + }); + } + + /** @type {Item} */ + const otherItem = { + source: 'Other', + wastedBytes: 0, + url: '', + totalBytes: 0, + urls: [], + sourceBytes: [], + }; + for (const item of items.filter(item => item.wastedBytes <= IGNORE_THRESHOLD_IN_BYTES)) { + otherItem.wastedBytes += item.wastedBytes; + for (let i = 0; i < item.urls.length; i++) { + const url = item.urls[i]; + if (!otherItem.urls.includes(url)) { + otherItem.urls.push(url); + } + } + items.splice(items.indexOf(item), 1); + } + if (otherItem.wastedBytes) { + items.push(otherItem); + } + + // TODO: explore a cutoff. + if (process.env.DEBUG) { + console.log(sourceDataAggregated.keys()); + + const all = sum(items); + // @ts-ignore + function sum(arr) { + // @ts-ignore + return arr.reduce((acc, cur) => acc + cur.wastedBytes, 0); + } + function print(x) { + const sum_ = sum(items.filter(item => item.wastedBytes >= x)); + console.log(x, sum_, (all - sum_) / all * 100); + } + for (let i = 0; i < 100; i += 10) { + print(i); + } + for (let i = 100; i < 1500; i += 100) { + print(i); + } + /* + initial thoughts: "0KB" is noisy in the report + + Could make an Other entry, but then that is unactionable. + + Just ignoring all the items is not a good idea b/c the sum of all the small items + can be meaningful - <500 bytes is ~5.5%. Is that too much to ignore? + + EDIT: oh, granularity is a thing. let's set that to 0.05 and make 100 bytes the threshold. + + https://www.coursehero.com/ + + 0 176188.36490136734 0 + 10 176188.36490136734 0 + 20 176188.36490136734 0 + 30 176141.61108744194 0.026536266428022284 + 40 176141.61108744194 0.026536266428022284 + 50 176141.61108744194 0.026536266428022284 + 60 176141.61108744194 0.026536266428022284 + 70 176141.61108744194 0.026536266428022284 + 80 176062.75412877792 0.07129345496778063 + 90 175975.1834931716 0.12099630319805638 + 100 175975.1834931716 0.12099630319805638 + 200 174014.05824632183 1.2340807273299335 + 300 172646.30987490347 2.010379645924272 + 400 169433.15980658625 3.834081267831022 + 500 166372.66452209078 5.5711399471647995 + 600 162028.34675652898 8.036863360849814 + 700 159503.6408045566 9.469821747963366 + 800 157215.92153878932 10.768272566238442 + 900 153868.20003692358 12.668353484600928 + 1000 153868.20003692358 12.668353484600928 + 1100 153868.20003692358 12.668353484600928 + 1200 152701.58106087992 13.330496513566967 + 1300 152701.58106087992 13.330496513566967 + 1400 151370.21055005147 14.086148290898443 + + */ + } + + /** @type {LH.Audit.Details.OpportunityColumnHeading[]} */ + const headings = [ + {key: 'source', valueType: 'code', subRows: {key: 'urls', valueType: 'url'}, label: str_(i18n.UIStrings.columnName)}, // TODO: or 'Source'? + {key: '_', valueType: 'bytes', subRows: {key: 'sourceBytes'}, granularity: 0.05, label: str_(i18n.UIStrings.columnSize)}, + {key: 'wastedBytes', valueType: 'bytes', granularity: 0.05, label: str_(i18n.UIStrings.columnWastedBytes)}, + ]; + + // TODO: show warning somewhere if no source maps. + return { + items, + headings, + wastedBytesByUrl, + }; + } +} + +module.exports = BundleDuplication; +module.exports.UIStrings = UIStrings; diff --git a/lighthouse-core/audits/byte-efficiency/byte-efficiency-audit.js b/lighthouse-core/audits/byte-efficiency/byte-efficiency-audit.js index 883bca533e8c..5400038e230e 100644 --- a/lighthouse-core/audits/byte-efficiency/byte-efficiency-audit.js +++ b/lighthouse-core/audits/byte-efficiency/byte-efficiency-audit.js @@ -27,6 +27,7 @@ const WASTED_MS_FOR_SCORE_OF_ZERO = 5000; /** * @typedef {object} ByteEfficiencyProduct * @property {Array} items + * @property {Map=} wastedBytesByUrl * @property {LH.Audit.Details.Opportunity['headings']} headings * @property {string} [displayValue] * @property {string} [explanation] @@ -65,7 +66,7 @@ class UnusedBytes extends Audit { * * @param {LH.Artifacts.NetworkRequest=} networkRecord * @param {number} totalBytes Uncompressed size of the resource - * @param {LH.Crdp.Network.ResourceType=} resourceType + * @param {LH.Crdp.Page.ResourceType=} resourceType * @return {number} */ static estimateTransferSize(networkRecord, totalBytes, resourceType) { @@ -135,7 +136,7 @@ class UnusedBytes extends Audit { * @param {Array} results The array of byte savings results per resource * @param {Node} graph * @param {Simulator} simulator - * @param {{includeLoad?: boolean, label?: string}=} options + * @param {{includeLoad?: boolean, label?: string, providedWastedBytesByUrl?: Map}=} options * @return {number} */ static computeWasteWithTTIGraph(results, graph, simulator, options) { @@ -144,10 +145,13 @@ class UnusedBytes extends Audit { const afterLabel = `${options.label}-after`; const simulationBeforeChanges = simulator.simulate(graph, {label: beforeLabel}); - /** @type {Map} */ - const resultsByUrl = new Map(); - for (const result of results) { - resultsByUrl.set(result.url, result); + + let wastedBytesByUrl = options.providedWastedBytesByUrl || new Map(); + if (!wastedBytesByUrl.size) { + wastedBytesByUrl = new Map(); + for (const {url, wastedBytes} of results) { + wastedBytesByUrl.set(url, (wastedBytesByUrl.get(url) || 0) + wastedBytes); + } } // Update all the transfer sizes to reflect implementing our recommendations @@ -155,13 +159,12 @@ class UnusedBytes extends Audit { const originalTransferSizes = new Map(); graph.traverse(node => { if (node.type !== 'network') return; - const result = resultsByUrl.get(node.record.url); - if (!result) return; + const wastedBytes = wastedBytesByUrl.get(node.record.url); + if (!wastedBytes) return; const original = node.record.transferSize; originalTransferSizes.set(node.record.requestId, original); - const wastedBytes = result.wastedBytes; node.record.transferSize = Math.max(original - wastedBytes, 0); }); @@ -197,7 +200,9 @@ class UnusedBytes extends Audit { const wastedBytes = results.reduce((sum, item) => sum + item.wastedBytes, 0); const wastedKb = Math.round(wastedBytes / KB_IN_BYTES); - const wastedMs = this.computeWasteWithTTIGraph(results, graph, simulator); + const wastedMs = this.computeWasteWithTTIGraph(results, graph, simulator, { + providedWastedBytesByUrl: result.wastedBytesByUrl, + }); let displayValue = result.displayValue || ''; if (typeof result.displayValue === 'undefined' && wastedBytes) { @@ -206,12 +211,27 @@ class UnusedBytes extends Audit { const details = Audit.makeOpportunityDetails(result.headings, results, wastedMs, wastedBytes); + console.dir({ + explanation: result.explanation, + warnings: result.warnings, + displayValue, + numericValue: wastedMs, + score: UnusedBytes.scoreForWastedMs(wastedMs), + extendedInfo: { + value: { + wastedMs, + wastedKb, + results, + }, + }, + details, + }); + return { explanation: result.explanation, warnings: result.warnings, displayValue, numericValue: wastedMs, - numericUnit: 'millisecond', score: UnusedBytes.scoreForWastedMs(wastedMs), extendedInfo: { value: { diff --git a/lighthouse-core/config/source-maps-config.js b/lighthouse-core/config/source-maps-config.js new file mode 100644 index 000000000000..4077b44c752e --- /dev/null +++ b/lighthouse-core/config/source-maps-config.js @@ -0,0 +1,31 @@ +/** + * @license Copyright 2020 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/** @type {LH.Config.Json} */ +const config = { + extends: 'lighthouse:default', + passes: [{ + passName: 'defaultPass', + gatherers: [ + 'source-maps', + ], + }], + audits: [ + 'byte-efficiency/bundle-duplication', + ], + // @ts-ignore: `title` is required in CategoryJson. setting to the same value as the default + // config is awkward - easier to omit the property here. Will defer to default config. + categories: { + 'performance': { + auditRefs: [ + {id: 'bundle-duplication', weight: 0, group: 'load-opportunities'}, + ], + }, + }, +}; + +module.exports = config; From 61a37dec2de42a8e1aefb81da921e6729a50414d Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Sat, 8 Feb 2020 22:13:57 -0800 Subject: [PATCH 02/21] update --- .../audits/byte-efficiency/bundle-duplication.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lighthouse-core/audits/byte-efficiency/bundle-duplication.js b/lighthouse-core/audits/byte-efficiency/bundle-duplication.js index af529a821c00..3baf5ad85ace 100644 --- a/lighthouse-core/audits/byte-efficiency/bundle-duplication.js +++ b/lighthouse-core/audits/byte-efficiency/bundle-duplication.js @@ -1,5 +1,5 @@ /** - * @license Copyright 2017 Google Inc. All Rights Reserved. + * @license Copyright 2020 Google Inc. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ @@ -107,13 +107,13 @@ class BundleDuplication extends ByteEfficiencyAudit { } /** - * @typedef ItemMulti + * @typedef ItemSubrows * @property {string[]} urls * @property {number[]} sourceBytes */ /** - * @typedef {LH.Audit.ByteEfficiencyItem & ItemMulti} Item + * @typedef {LH.Audit.ByteEfficiencyItem & ItemSubrows} Item */ /** @type {Item[]} */ From 9d6d6266129b7941f968813c4192cfcfbb88d675 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 13 Feb 2020 15:01:59 -0800 Subject: [PATCH 03/21] config --- lighthouse-core/config/experimental-config.js | 6 ++-- lighthouse-core/config/source-maps-config.js | 31 ------------------- 2 files changed, 2 insertions(+), 35 deletions(-) delete mode 100644 lighthouse-core/config/source-maps-config.js diff --git a/lighthouse-core/config/experimental-config.js b/lighthouse-core/config/experimental-config.js index 134d7c487085..d838980f35a0 100644 --- a/lighthouse-core/config/experimental-config.js +++ b/lighthouse-core/config/experimental-config.js @@ -20,16 +20,14 @@ const config = { ], }], audits: [ - // About to be added. - // 'bundle-duplication', + 'byte-efficiency/bundle-duplication', ], // @ts-ignore: `title` is required in CategoryJson. setting to the same value as the default // config is awkward - easier to omit the property here. Will defer to default config. categories: { 'performance': { auditRefs: [ - // About to be added. - // {id: 'bundle-duplication', weight: 0, group: 'load-opportunities'}, + {id: 'bundle-duplication', weight: 0, group: 'load-opportunities'}, ], }, }, diff --git a/lighthouse-core/config/source-maps-config.js b/lighthouse-core/config/source-maps-config.js deleted file mode 100644 index 4077b44c752e..000000000000 --- a/lighthouse-core/config/source-maps-config.js +++ /dev/null @@ -1,31 +0,0 @@ -/** - * @license Copyright 2020 Google Inc. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - */ -'use strict'; - -/** @type {LH.Config.Json} */ -const config = { - extends: 'lighthouse:default', - passes: [{ - passName: 'defaultPass', - gatherers: [ - 'source-maps', - ], - }], - audits: [ - 'byte-efficiency/bundle-duplication', - ], - // @ts-ignore: `title` is required in CategoryJson. setting to the same value as the default - // config is awkward - easier to omit the property here. Will defer to default config. - categories: { - 'performance': { - auditRefs: [ - {id: 'bundle-duplication', weight: 0, group: 'load-opportunities'}, - ], - }, - }, -}; - -module.exports = config; From 8320059f2d01a2542de45d8fb2ec283acb4d8163 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Sat, 29 Feb 2020 14:52:12 -0800 Subject: [PATCH 04/21] add test for computed artifact --- .../byte-efficiency/bundle-duplication.js | 72 +----------- .../computed/javascript-duplication.js | 104 ++++++++++++++++++ .../computed/javascript-duplication-test.js | 83 ++++++++++++++ 3 files changed, 192 insertions(+), 67 deletions(-) create mode 100644 lighthouse-core/computed/javascript-duplication.js create mode 100644 lighthouse-core/test/computed/javascript-duplication-test.js diff --git a/lighthouse-core/audits/byte-efficiency/bundle-duplication.js b/lighthouse-core/audits/byte-efficiency/bundle-duplication.js index 3baf5ad85ace..c4b4a6cb4911 100644 --- a/lighthouse-core/audits/byte-efficiency/bundle-duplication.js +++ b/lighthouse-core/audits/byte-efficiency/bundle-duplication.js @@ -6,13 +6,13 @@ 'use strict'; const ByteEfficiencyAudit = require('./byte-efficiency-audit.js'); -const JsBundles = require('../../computed/js-bundles.js'); +const JavascriptDuplication = require('../../computed/javascript-duplication.js'); const i18n = require('../../lib/i18n/i18n.js'); // TODO: write these. const UIStrings = { /** Imperative title of a Lighthouse audit that tells the user to remove content from their CSS that isn’t needed immediately and instead load that content at a later time. This is displayed in a list of audit titles that Lighthouse generates. */ - title: 'Remove duplicated code within bundles', + title: 'Remove duplicated modules in JavaScript bundles', /** Description of a Lighthouse audit that tells the user *why* they should defer loading any content in CSS that isn’t needed at page load. This is displayed after a user expands the section to see more. No word length limits. 'Learn More' becomes link text to additional documentation. */ description: 'Remove dead rules from stylesheets and defer the loading of CSS not used for ' + 'above-the-fold content to reduce unnecessary bytes consumed by network activity. ' + @@ -44,67 +44,7 @@ class BundleDuplication extends ByteEfficiencyAudit { * @return {Promise} */ static async audit_(artifacts, networkRecords, context) { - const bundles = await JsBundles.request(artifacts, context); - - /** - * @typedef SourceData - * @property {string} normalizedSource - * @property {number} size - */ - - /** @type {Map} */ - const sourceDatasMap = new Map(); - - // Determine size of each `sources` entry. - for (const {rawMap, sizes} of bundles) { - /** @type {SourceData[]} */ - const sourceDatas = []; - sourceDatasMap.set(rawMap, sourceDatas); - - for (let i = 0; i < rawMap.sources.length; i++) { - const source = rawMap.sources[i]; - // Trim trailing question mark - b/c webpack. - let normalizedSource = source.replace(/\?$/, ''); - // Normalize paths for dependencies by keeping everything after the last `node_modules`. - const lastNodeModulesIndex = normalizedSource.lastIndexOf('node_modules'); - if (lastNodeModulesIndex !== -1) { - normalizedSource = source.substring(lastNodeModulesIndex); - } - - // Ignore bundle overhead. - if (normalizedSource.includes('webpack/bootstrap')) continue; - if (normalizedSource.includes('(webpack)/buildin')) continue; - // Ignore shims. - if (normalizedSource.includes('external ')) continue; - - const fullSource = (rawMap.sourceRoot || '') + source; - const sourceSize = sizes.files[fullSource]; - - sourceDatas.push({ - normalizedSource, - size: sourceSize, - }); - } - } - - /** @type {Map>} */ - const sourceDataAggregated = new Map(); - for (const {rawMap, script} of bundles) { - const sourceDatas = sourceDatasMap.get(rawMap); - if (!sourceDatas) continue; - - for (const sourceData of sourceDatas) { - let data = sourceDataAggregated.get(sourceData.normalizedSource); - if (!data) { - data = []; - sourceDataAggregated.set(sourceData.normalizedSource, data); - } - data.push({ - scriptUrl: script.src || '', - size: sourceData.size, - }); - } - } + const sourceDataAggregated = await JavascriptDuplication.request(artifacts, context); /** * @typedef ItemSubrows @@ -121,9 +61,7 @@ class BundleDuplication extends ByteEfficiencyAudit { /** @type {Map} */ const wastedBytesByUrl = new Map(); - for (const [key, sourceDatas] of sourceDataAggregated.entries()) { - if (sourceDatas.length === 1) continue; - + for (const [source, sourceDatas] of sourceDataAggregated.entries()) { // One copy of this module is treated as the canonical version - the rest will have // non-zero `wastedBytes`. In the case of all copies being the same version, all sizes are // equal and the selection doesn't matter. When the copies are different versions, it does @@ -147,7 +85,7 @@ class BundleDuplication extends ByteEfficiencyAudit { } items.push({ - source: key, + source, wastedBytes: wastedBytesTotal, // Not needed, but keeps typescript happy. url: '', diff --git a/lighthouse-core/computed/javascript-duplication.js b/lighthouse-core/computed/javascript-duplication.js new file mode 100644 index 000000000000..1e795c23889d --- /dev/null +++ b/lighthouse-core/computed/javascript-duplication.js @@ -0,0 +1,104 @@ +/** + * @license Copyright 2020 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const makeComputedArtifact = require('./computed-artifact.js'); +const JsBundles = require('./js-bundles.js'); + +class JavascriptDuplication { + /** + * @param {string} source + */ + static _normalizeSource(source) { + // Trim trailing question mark - b/c webpack. + source = source.replace(/\?$/, ''); + + // Normalize paths for dependencies by keeping everything after the last `node_modules`. + const lastNodeModulesIndex = source.lastIndexOf('node_modules'); + if (lastNodeModulesIndex !== -1) { + source = source.substring(lastNodeModulesIndex); + } + + return source; + } + + /** + * @param {string} source + */ + static _shouldIgnoreSource(source) { + // Ignore bundle overhead. + if (source.includes('webpack/bootstrap')) return true; + if (source.includes('(webpack)/buildin')) return true; + + // Ignore shims. + if (source.includes('external ')) return true; + + return false; + } + + /** + * @param {LH.Artifacts} artifacts + * @param {LH.Audit.Context} context + */ + static async compute_(artifacts, context) { + const bundles = await JsBundles.request(artifacts, context); + + /** + * @typedef SourceData + * @property {string} source + * @property {number} size + */ + + /** @type {Map} */ + const sourceDatasMap = new Map(); + + // Determine size of each `sources` entry. + for (const {rawMap, sizes} of bundles) { + /** @type {SourceData[]} */ + const sourceDatas = []; + sourceDatasMap.set(rawMap, sourceDatas); + + for (let i = 0; i < rawMap.sources.length; i++) { + const source = JavascriptDuplication._normalizeSource(rawMap.sources[i]); + if (this._shouldIgnoreSource(source)) continue; + + const fullSource = (rawMap.sourceRoot || '') + source; + const sourceSize = sizes.files[fullSource]; + sourceDatas.push({ + source, + size: sourceSize, + }); + } + } + + /** @type {Map>} */ + const sourceDataAggregated = new Map(); + for (const {rawMap, script} of bundles) { + const sourceDatas = sourceDatasMap.get(rawMap); + if (!sourceDatas) continue; + + for (const sourceData of sourceDatas) { + let data = sourceDataAggregated.get(sourceData.source); + if (!data) { + data = []; + sourceDataAggregated.set(sourceData.source, data); + } + data.push({ + scriptUrl: script.src || '', + size: sourceData.size, + }); + } + } + + for (const [key, value] of sourceDataAggregated.entries()) { + if (value.length === 1) sourceDataAggregated.delete(key); + } + + return sourceDataAggregated; + } +} + +module.exports = makeComputedArtifact(JavascriptDuplication); diff --git a/lighthouse-core/test/computed/javascript-duplication-test.js b/lighthouse-core/test/computed/javascript-duplication-test.js new file mode 100644 index 000000000000..20ae51b0d53c --- /dev/null +++ b/lighthouse-core/test/computed/javascript-duplication-test.js @@ -0,0 +1,83 @@ +/** + * @license Copyright 2020 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/* eslint-env jest */ + +const assert = require('assert'); +const fs = require('fs'); +const JavascriptDuplication = require('../../computed/javascript-duplication.js'); + +function load(name) { + const mapJson = fs.readFileSync(`${__dirname}/../fixtures/source-maps/${name}.js.map`, 'utf-8'); + const content = fs.readFileSync(`${__dirname}/../fixtures/source-maps/${name}.js`, 'utf-8'); + return {map: JSON.parse(mapJson), content}; +} + +describe('JavascriptDuplication computed artifact', () => { + it('works', async () => { + const context = {computedCache: new Map()}; + const {map, content} = load('foo.min'); + const artifacts = { + SourceMaps: [ + {scriptUrl: 'https://example.com/foo1.min.js', map}, + {scriptUrl: 'https://example.com/foo2.min.js', map}, + ], + ScriptElements: [ + {src: 'https://example.com/foo1.min.js', content}, + {src: 'https://example.com/foo2.min.js', content}, + ], + }; + const results = await JavascriptDuplication.request(artifacts, context); + expect(results).toMatchInlineSnapshot(` + Map { + "node_modules/browser-pack/_prelude.js" => Array [ + Object { + "scriptUrl": "https://example.com/foo1.min.js", + "size": 480, + }, + Object { + "scriptUrl": "https://example.com/foo2.min.js", + "size": 480, + }, + ], + "src/bar.js" => Array [ + Object { + "scriptUrl": "https://example.com/foo1.min.js", + "size": 104, + }, + Object { + "scriptUrl": "https://example.com/foo2.min.js", + "size": 104, + }, + ], + "src/foo.js" => Array [ + Object { + "scriptUrl": "https://example.com/foo1.min.js", + "size": 98, + }, + Object { + "scriptUrl": "https://example.com/foo2.min.js", + "size": 98, + }, + ], + } + `); + }); + + it('_normalizeSource', () => { + const testCases = [ + ['test.js', 'test.js'], + ['node_modules/othermodule.js', 'node_modules/othermodule.js'], + ['node_modules/somemodule/node_modules/othermodule.js', 'node_modules/othermodule.js'], + ['node_modules/somemodule/node_modules/somemodule2/node_modules/othermodule.js', 'node_modules/othermodule.js'], + ['webpack.js?', 'webpack.js'], + ]; + for (const [input, expected] of testCases) { + expect(JavascriptDuplication._normalizeSource(input)).toBe(expected); + } + }); +}); From 8fc34b3f6b3e250759158b9b2f7c3c215d8a80f8 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Mon, 2 Mar 2020 10:07:14 -0800 Subject: [PATCH 05/21] fix size lookup --- lighthouse-core/computed/javascript-duplication.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lighthouse-core/computed/javascript-duplication.js b/lighthouse-core/computed/javascript-duplication.js index 1e795c23889d..f4e298bf4fbc 100644 --- a/lighthouse-core/computed/javascript-duplication.js +++ b/lighthouse-core/computed/javascript-duplication.js @@ -62,13 +62,12 @@ class JavascriptDuplication { sourceDatasMap.set(rawMap, sourceDatas); for (let i = 0; i < rawMap.sources.length; i++) { - const source = JavascriptDuplication._normalizeSource(rawMap.sources[i]); - if (this._shouldIgnoreSource(source)) continue; + if (this._shouldIgnoreSource(rawMap.sources[i])) continue; - const fullSource = (rawMap.sourceRoot || '') + source; + const fullSource = (rawMap.sourceRoot || '') + rawMap.sources[i]; const sourceSize = sizes.files[fullSource]; sourceDatas.push({ - source, + source: JavascriptDuplication._normalizeSource(rawMap.sources[i]), size: sourceSize, }); } From 8b2671dfb347d27f8405b79b7e9e75fed3032421 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Mon, 2 Mar 2020 10:30:53 -0800 Subject: [PATCH 06/21] complex maps --- .../computed/javascript-duplication-test.js | 498 +++++++++++++++++- .../source-maps/coursehero-bundle-1.js | 2 + .../source-maps/coursehero-bundle-1.js.map | 1 + .../source-maps/coursehero-bundle-2.js | 2 + .../source-maps/coursehero-bundle-2.js.map | 1 + 5 files changed, 502 insertions(+), 2 deletions(-) create mode 100644 lighthouse-core/test/fixtures/source-maps/coursehero-bundle-1.js create mode 100644 lighthouse-core/test/fixtures/source-maps/coursehero-bundle-1.js.map create mode 100644 lighthouse-core/test/fixtures/source-maps/coursehero-bundle-2.js create mode 100644 lighthouse-core/test/fixtures/source-maps/coursehero-bundle-2.js.map diff --git a/lighthouse-core/test/computed/javascript-duplication-test.js b/lighthouse-core/test/computed/javascript-duplication-test.js index 20ae51b0d53c..a392a8c00ac5 100644 --- a/lighthouse-core/test/computed/javascript-duplication-test.js +++ b/lighthouse-core/test/computed/javascript-duplication-test.js @@ -18,7 +18,7 @@ function load(name) { } describe('JavascriptDuplication computed artifact', () => { - it('works', async () => { + it('works (simple)', async () => { const context = {computedCache: new Map()}; const {map, content} = load('foo.min'); const artifacts = { @@ -68,12 +68,506 @@ describe('JavascriptDuplication computed artifact', () => { `); }); + it('works (complex)', async () => { + const context = {computedCache: new Map()}; + const bundleData1 = load('coursehero-bundle-1'); + const bundleData2 = load('coursehero-bundle-2'); + const artifacts = { + SourceMaps: [ + {scriptUrl: 'https://example.com/coursehero-bundle-1.js', map: bundleData1.map}, + {scriptUrl: 'https://example.com/coursehero-bundle-2.js', map: bundleData2.map}, + ], + ScriptElements: [ + {src: 'https://example.com/coursehero-bundle-1.js', content: bundleData1.content}, + {src: 'https://example.com/coursehero-bundle-2.js', content: bundleData2.content}, + ], + }; + const results = await JavascriptDuplication.request(artifacts, context); + expect(results).toMatchInlineSnapshot(` + Map { + "Control/assets/js/vendor/ng/select/select.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 48513, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 48513, + }, + ], + "Control/assets/js/vendor/ng/select/angular-sanitize.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 9135, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 9135, + }, + ], + "node_modules/@babel/runtime/helpers/classCallCheck.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 358, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 236, + }, + ], + "node_modules/@babel/runtime/helpers/createClass.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 799, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 496, + }, + ], + "node_modules/@babel/runtime/helpers/assertThisInitialized.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 294, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 296, + }, + ], + "node_modules/@babel/runtime/helpers/applyDecoratedDescriptor.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 892, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 446, + }, + ], + "node_modules/@babel/runtime/helpers/possibleConstructorReturn.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 228, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 230, + }, + ], + "node_modules/@babel/runtime/helpers/getPrototypeOf.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 338, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 361, + }, + ], + "node_modules/@babel/runtime/helpers/inherits.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 528, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 528, + }, + ], + "node_modules/@babel/runtime/helpers/defineProperty.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 288, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 290, + }, + ], + "node_modules/@babel/runtime/helpers/extends.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 490, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 245, + }, + ], + "node_modules/@babel/runtime/helpers/typeof.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 992, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 992, + }, + ], + "node_modules/@babel/runtime/helpers/setPrototypeOf.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 260, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 290, + }, + ], + "js/src/common/base-component.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 459, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 216, + }, + ], + "js/src/utils/service/amplitude-service.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 1348, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 1325, + }, + ], + "js/src/aged-beef.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 213, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 194, + }, + ], + "js/src/utils/service/api-service.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 116, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 54, + }, + ], + "js/src/common/decorators/throttle.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 251, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 244, + }, + ], + "js/src/utils/service/gsa-inmeta-tags.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 591, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 563, + }, + ], + "js/src/utils/service/global-service.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 336, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 167, + }, + ], + "js/src/search/results/store/filter-actions.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 946, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 956, + }, + ], + "js/src/search/results/store/item/resource-types.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 783, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 775, + }, + ], + "js/src/common/input/keycode.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 237, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 223, + }, + ], + "js/src/search/results/store/filter-store.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 12717, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 12650, + }, + ], + "js/src/search/results/view/filter/autocomplete-list.tsx" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 1134, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 1143, + }, + ], + "js/src/search/results/view/filter/autocomplete-filter.tsx" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 3823, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 3812, + }, + ], + "js/src/search/results/view/filter/autocomplete-filter-with-icon.tsx" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 2696, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 2693, + }, + ], + "js/src/search/results/service/api/filter-api-service.ts" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 554, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 534, + }, + ], + "js/src/common/component/school-search.tsx" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 5316, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 5840, + }, + ], + "js/src/common/component/search/abstract-taxonomy-search.tsx" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 3103, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 3098, + }, + ], + "js/src/common/component/search/course-search.tsx" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 544, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 545, + }, + ], + "node_modules/lodash-es/_freeGlobal.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 93, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 118, + }, + ], + "node_modules/lodash-es/_root.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 93, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 93, + }, + ], + "node_modules/lodash-es/_Symbol.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 10, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 10, + }, + ], + "node_modules/lodash-es/_arrayMap.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 99, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 99, + }, + ], + "node_modules/lodash-es/isArray.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 16, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 16, + }, + ], + "node_modules/lodash-es/_getRawTag.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 206, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 206, + }, + ], + "node_modules/lodash-es/_objectToString.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 64, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 64, + }, + ], + "node_modules/lodash-es/_baseGetTag.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 143, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 143, + }, + ], + "node_modules/lodash-es/isObjectLike.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 54, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 54, + }, + ], + "node_modules/lodash-es/isSymbol.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 79, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 79, + }, + ], + "node_modules/lodash-es/_baseToString.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 198, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 198, + }, + ], + "node_modules/lodash-es/isObject.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 79, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 80, + }, + ], + "node_modules/lodash-es/toNumber.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 354, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 370, + }, + ], + "node_modules/lodash-es/toFinite.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 117, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 118, + }, + ], + "node_modules/lodash-es/toInteger.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 60, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 60, + }, + ], + "node_modules/lodash-es/toString.js" => Array [ + Object { + "scriptUrl": "https://example.com/coursehero-bundle-1.js", + "size": 43, + }, + Object { + "scriptUrl": "https://example.com/coursehero-bundle-2.js", + "size": 43, + }, + ], + } + `); + }); + it('_normalizeSource', () => { const testCases = [ ['test.js', 'test.js'], ['node_modules/othermodule.js', 'node_modules/othermodule.js'], ['node_modules/somemodule/node_modules/othermodule.js', 'node_modules/othermodule.js'], - ['node_modules/somemodule/node_modules/somemodule2/node_modules/othermodule.js', 'node_modules/othermodule.js'], + [ + 'node_modules/somemodule/node_modules/somemodule2/node_modules/othermodule.js', + 'node_modules/othermodule.js', + ], ['webpack.js?', 'webpack.js'], ]; for (const [input, expected] of testCases) { diff --git a/lighthouse-core/test/fixtures/source-maps/coursehero-bundle-1.js b/lighthouse-core/test/fixtures/source-maps/coursehero-bundle-1.js new file mode 100644 index 000000000000..bb26327259f2 --- /dev/null +++ b/lighthouse-core/test/fixtures/source-maps/coursehero-bundle-1.js @@ -0,0 +1,2 @@ +(function($){var _={isMsie:function(){return/(msie|trident)/i.test(navigator.userAgent)?navigator.userAgent.match(/(msie |rv:)(\d+(.\d+)?)/i)[2]:false},isBlankString:function(str){return!str||/^\s*$/.test(str)},escapeRegExChars:function(str){return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g,"\\$&")},isString:function(obj){return typeof obj==="string"},isNumber:function(obj){return typeof obj==="number"},isArray:$.isArray,isFunction:$.isFunction,isObject:$.isPlainObject,isUndefined:function(obj){return typeof obj==="undefined"},bind:$.proxy,each:function(collection,cb){$.each(collection,reverseArgs);function reverseArgs(index,value){return cb(value,index)}},map:$.map,filter:$.grep,every:function(obj,test){var result=true;if(!obj){return result}$.each(obj,function(key,val){if(!(result=test.call(null,val,key,obj))){return false}});return!!result},some:function(obj,test){var result=false;if(!obj){return result}$.each(obj,function(key,val){if(result=test.call(null,val,key,obj)){return false}});return!!result},mixin:$.extend,getUniqueId:function(){var counter=0;return function(){return counter++}}(),templatify:function templatify(obj){return $.isFunction(obj)?obj:template;function template(){return String(obj)}},defer:function(fn){setTimeout(fn,0)},debounce:function(func,wait,immediate){var timeout,result;return function(){var context=this,args=arguments,later,callNow;later=function(){timeout=null;if(!immediate){result=func.apply(context,args)}};callNow=immediate&&!timeout;clearTimeout(timeout);timeout=setTimeout(later,wait);if(callNow){result=func.apply(context,args)}return result}},throttle:function(func,wait){var context,args,timeout,result,previous,later;previous=0;later=function(){previous=new Date;timeout=null;result=func.apply(context,args)};return function(){var now=new Date,remaining=wait-(now-previous);context=this;args=arguments;if(remaining<=0){clearTimeout(timeout);timeout=null;previous=now;result=func.apply(context,args)}else if(!timeout){timeout=setTimeout(later,remaining)}return result}},noop:function(){}};var VERSION="0.10.2";var tokenizers=function(root){return{nonword:nonword,whitespace:whitespace,obj:{nonword:getObjTokenizer(nonword),whitespace:getObjTokenizer(whitespace)}};function whitespace(s){return s.split(/\s+/)}function nonword(s){return s.split(/\W+/)}function getObjTokenizer(tokenizer){return function setKey(key){return function tokenize(o){return tokenizer(o[key])}}}}();var LruCache=function(){function LruCache(maxSize){this.maxSize=maxSize||100;this.size=0;this.hash={};this.list=new List}_.mixin(LruCache.prototype,{set:function set(key,val){var tailItem=this.list.tail,node;if(this.size>=this.maxSize){this.list.remove(tailItem);delete this.hash[tailItem.key]}if(node=this.hash[key]){node.val=val;this.list.moveToFront(node)}else{node=new Node(key,val);this.list.add(node);this.hash[key]=node;this.size++}},get:function get(key){var node=this.hash[key];if(node){this.list.moveToFront(node);return node.val}}});function List(){this.head=this.tail=null}_.mixin(List.prototype,{add:function add(node){if(this.head){node.next=this.head;this.head.prev=node}this.head=node;this.tail=this.tail||node},remove:function remove(node){node.prev?node.prev.next=node.next:this.head=node.next;node.next?node.next.prev=node.prev:this.tail=node.prev},moveToFront:function(node){this.remove(node);this.add(node)}});function Node(key,val){this.key=key;this.val=val;this.prev=this.next=null}return LruCache}();var PersistentStorage=function(){var ls,methods;try{ls=window.localStorage;ls.setItem("~~~","!");ls.removeItem("~~~")}catch(err){ls=null}function PersistentStorage(namespace){this.prefix=["__",namespace,"__"].join("");this.ttlKey="__ttl__";this.keyMatcher=new RegExp("^"+this.prefix)}if(ls&&window.JSON){methods={_prefix:function(key){return this.prefix+key},_ttlKey:function(key){return this._prefix(key)+this.ttlKey},get:function(key){if(this.isExpired(key)){this.remove(key)}return decode(ls.getItem(this._prefix(key)))},set:function(key,val,ttl){if(_.isNumber(ttl)){ls.setItem(this._ttlKey(key),encode(now()+ttl))}else{ls.removeItem(this._ttlKey(key))}return ls.setItem(this._prefix(key),encode(val))},remove:function(key){ls.removeItem(this._ttlKey(key));ls.removeItem(this._prefix(key));return this},clear:function(){var i,key,keys=[],len=ls.length;for(i=0;ittl?true:false}}}else{methods={get:_.noop,set:_.noop,remove:_.noop,clear:_.noop,isExpired:_.noop}}_.mixin(PersistentStorage.prototype,methods);return PersistentStorage;function now(){return(new Date).getTime()}function encode(val){return JSON.stringify(_.isUndefined(val)?null:val)}function decode(val){return JSON.parse(val)}}();var Transport=function(){var pendingRequestsCount=0,pendingRequests={},maxPendingRequests=6,requestCache=new LruCache(10);function Transport(o){o=o||{};this._send=o.transport?callbackToDeferred(o.transport):$.ajax;this._get=o.rateLimiter?o.rateLimiter(this._get):this._get}Transport.setMaxPendingRequests=function setMaxPendingRequests(num){maxPendingRequests=num};Transport.resetCache=function clearCache(){requestCache=new LruCache(10)};_.mixin(Transport.prototype,{_get:function(url,o,cb){var that=this,jqXhr;if(jqXhr=pendingRequests[url]){jqXhr.done(done).fail(fail)}else if(pendingRequestsCountarrayB[bi]){bi++}else{intersection.push(arrayA[ai]);ai++;bi++}}return intersection;function compare(a,b){return a-b}}}();var oParser=function(){return{local:getLocal,prefetch:getPrefetch,remote:getRemote};function getLocal(o){return o.local||null}function getPrefetch(o){var prefetch,defaults;defaults={url:null,thumbprint:"",ttl:24*60*60*1e3,filter:null,ajax:{}};if(prefetch=o.prefetch||null){prefetch=_.isString(prefetch)?{url:prefetch}:prefetch;prefetch=_.mixin(defaults,prefetch);prefetch.thumbprint=VERSION+prefetch.thumbprint;prefetch.ajax.type=prefetch.ajax.type||"GET";prefetch.ajax.dataType=prefetch.ajax.dataType||"json";!prefetch.url&&$.error("prefetch requires url to be set")}return prefetch}function getRemote(o){var remote,defaults;defaults={url:null,wildcard:"%QUERY",replace:null,rateLimitBy:"debounce",rateLimitWait:300,send:null,filter:null,ajax:{}};if(remote=o.remote||null){remote=_.isString(remote)?{url:remote}:remote;remote=_.mixin(defaults,remote);remote.rateLimiter=/^throttle$/i.test(remote.rateLimitBy)?byThrottle(remote.rateLimitWait):byDebounce(remote.rateLimitWait);remote.ajax.type=remote.ajax.type||"GET";remote.ajax.dataType=remote.ajax.dataType||"json";delete remote.rateLimitBy;delete remote.rateLimitWait;!remote.url&&$.error("remote requires url to be set")}return remote;function byDebounce(wait){return function(fn){return _.debounce(fn,wait)}}function byThrottle(wait){return function(fn){return _.throttle(fn,wait)}}}}();(function(root){var old,keys;old=root.Bloodhound;keys={data:"data",protocol:"protocol",thumbprint:"thumbprint"};root.Bloodhound=Bloodhound;function Bloodhound(o){if(!o||!o.local&&!o.prefetch&&!o.remote){$.error("one of local, prefetch, or remote is required")}this.limit=o.limit||5;this.sorter=getSorter(o.sorter);this.dupDetector=o.dupDetector||ignoreDuplicates;this.local=oParser.local(o);this.prefetch=oParser.prefetch(o);this.remote=oParser.remote(o);this.cacheKey=this.prefetch?this.prefetch.cacheKey||this.prefetch.url:null;this.index=new SearchIndex({datumTokenizer:o.datumTokenizer,queryTokenizer:o.queryTokenizer});this.storage=this.cacheKey?new PersistentStorage(this.cacheKey):null}Bloodhound.noConflict=function noConflict(){root.Bloodhound=old;return Bloodhound};Bloodhound.tokenizers=tokenizers;_.mixin(Bloodhound.prototype,{_loadPrefetch:function loadPrefetch(o){var that=this,serialized,deferred;if(serialized=this._readFromStorage(o.thumbprint)){this.index.bootstrap(serialized);deferred=$.Deferred().resolve()}else{deferred=$.ajax(o.url,o.ajax).done(handlePrefetchResponse)}return deferred;function handlePrefetchResponse(resp){that.clear();that.add(o.filter?o.filter(resp):resp);that._saveToStorage(that.index.serialize(),o.thumbprint,o.ttl)}},_getFromRemote:function getFromRemote(query,cb){var that=this,url,uriEncodedQuery;query=query||"";uriEncodedQuery=encodeURIComponent(query);url=this.remote.replace?this.remote.replace(this.remote.url,query):this.remote.url.replace(this.remote.wildcard,uriEncodedQuery);return this.transport.get(url,this.remote.ajax,handleRemoteResponse);function handleRemoteResponse(err,resp){err?cb([]):cb(that.remote.filter?that.remote.filter(resp):resp)}},_saveToStorage:function saveToStorage(data,thumbprint,ttl){if(this.storage){this.storage.set(keys.data,data,ttl);this.storage.set(keys.protocol,location.protocol,ttl);this.storage.set(keys.thumbprint,thumbprint,ttl)}},_readFromStorage:function readFromStorage(thumbprint){var stored={},isExpired;if(this.storage){stored.data=this.storage.get(keys.data);stored.protocol=this.storage.get(keys.protocol);stored.thumbprint=this.storage.get(keys.thumbprint)}isExpired=stored.thumbprint!==thumbprint||stored.protocol!==location.protocol;return stored.data&&!isExpired?stored.data:null},_initialize:function initialize(){var that=this,local=this.local,deferred;deferred=this.prefetch?this._loadPrefetch(this.prefetch):$.Deferred().resolve();local&&deferred.done(addLocalToIndex);this.transport=this.remote?new Transport(this.remote):null;return this.initPromise=deferred.promise();function addLocalToIndex(){that.add(_.isFunction(local)?local():local)}},initialize:function initialize(force){return!this.initPromise||force?this._initialize():this.initPromise},add:function add(data){this.index.add(data)},get:function get(query,cb){var that=this,matches=[],cacheHit=false;matches=this.index.get(query);matches=this.sorter(matches).slice(0,this.limit);if(matches.length0||!this.transport)&&cb&&cb(matches)}function returnRemoteMatches(remoteMatches){var matchesWithBackfill=matches.slice(0);_.each(remoteMatches,function(remoteMatch){var isDuplicate;isDuplicate=_.some(matchesWithBackfill,function(match){return that.dupDetector(remoteMatch,match)});!isDuplicate&&matchesWithBackfill.push(remoteMatch);return matchesWithBackfill.length