From 9e0e9dfb67a3dd38b611b02d1129b197da8a2b0c Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Tue, 6 Oct 2020 17:57:15 -0500 Subject: [PATCH] new_audit: add script-treemap-data to experimental (#11271) --- lighthouse-core/audits/script-treemap-data.js | 264 ++++++++ .../computed/module-duplication.js | 4 +- lighthouse-core/computed/resource-summary.js | 1 + lighthouse-core/config/experimental-config.js | 1 + .../script-treemap-data-test.js.snap | 590 ++++++++++++++++++ .../duplicated-javascript-test.js | 21 +- .../byte-efficiency/unused-javascript-test.js | 41 +- .../test/audits/script-treemap-data-test.js | 301 +++++++++ .../test/audits/valid-source-maps-test.js | 17 +- .../test/computed/js-bundles-test.js | 16 +- .../test/computed/module-duplication-test.js | 18 +- .../test/gather/gather-runner-test.js | 10 +- lighthouse-core/test/test-utils.js | 65 +- tsconfig.json | 1 + 14 files changed, 1260 insertions(+), 90 deletions(-) create mode 100644 lighthouse-core/audits/script-treemap-data.js create mode 100644 lighthouse-core/test/audits/__snapshots__/script-treemap-data-test.js.snap create mode 100644 lighthouse-core/test/audits/script-treemap-data-test.js diff --git a/lighthouse-core/audits/script-treemap-data.js b/lighthouse-core/audits/script-treemap-data.js new file mode 100644 index 000000000000..24f42e54b3f5 --- /dev/null +++ b/lighthouse-core/audits/script-treemap-data.js @@ -0,0 +1,264 @@ +/** + * @license Copyright 2020 The Lighthouse Authors. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/** + * @fileoverview + * Creates treemap data for treemap app. + */ + +const Audit = require('./audit.js'); +const JsBundles = require('../computed/js-bundles.js'); +const UnusedJavaScriptSummary = require('../computed/unused-javascript-summary.js'); +const ModuleDuplication = require('../computed/module-duplication.js'); + +/** + * @typedef {RootNodeContainer[]} TreemapData + */ + +/** + * Ex: https://gist.github.com/connorjclark/0ef1099ae994c075e36d65fecb4d26a7 + * @typedef RootNodeContainer + * @property {string} name Arbitrary name identifier. Usually a script url. + * @property {Node} node + */ + +/** + * @typedef Node + * @property {string} name Arbitrary name identifier. Usually a path component from a source map. + * @property {number} resourceBytes + * @property {number=} unusedBytes + * @property {string=} duplicatedNormalizedModuleName If present, this module is a duplicate. String is normalized source path. See ModuleDuplication.normalizeSource + * @property {Node[]=} children + */ + +/** + * @typedef {Omit} SourceData + */ + +class ScriptTreemapDataAudit extends Audit { + /** + * @return {LH.Audit.Meta} + */ + static get meta() { + return { + id: 'script-treemap-data', + scoreDisplayMode: Audit.SCORING_MODES.INFORMATIVE, + title: 'Script Treemap Data', + description: 'Used for treemap app', + requiredArtifacts: + ['traces', 'devtoolsLogs', 'SourceMaps', 'ScriptElements', 'JsUsage', 'URL'], + }; + } + + /** + * Returns a tree data structure where leaf nodes are sources (ie. real files from source tree) + * from a source map, and non-leaf nodes are directories. Leaf nodes have data + * for bytes, coverage, etc., when available, and non-leaf nodes have the + * same data as the sum of all descendant leaf nodes. + * @param {string} sourceRoot + * @param {Record} sourcesData + * @return {Node} + */ + static prepareTreemapNodes(sourceRoot, sourcesData) { + /** + * @param {string} name + * @return {Node} + */ + function newNode(name) { + return { + name, + resourceBytes: 0, + }; + } + + const topNode = newNode(sourceRoot); + + /** + * Given a slash-delimited path, traverse the Node structure and increment + * the data provided for each node in the chain. Creates nodes as needed. + * Ex: path/to/file.js will find or create "path" on `node`, increment the data fields, + * and continue with "to", and so on. + * @param {string} source + * @param {SourceData} data + */ + function addAllNodesInSourcePath(source, data) { + let node = topNode; + + // Apply the data to the topNode. + topNode.resourceBytes += data.resourceBytes; + if (data.unusedBytes) topNode.unusedBytes = (topNode.unusedBytes || 0) + data.unusedBytes; + + // Strip off the shared root. + const sourcePathSegments = source.replace(sourceRoot, '').split(/\/+/); + sourcePathSegments.forEach((sourcePathSegment, i) => { + const isLeaf = i === sourcePathSegments.length - 1; + + let child = node.children && node.children.find(child => child.name === sourcePathSegment); + if (!child) { + child = newNode(sourcePathSegment); + node.children = node.children || []; + node.children.push(child); + } + node = child; + + // Now that we've found or created the next node in the path, apply the data. + node.resourceBytes += data.resourceBytes; + if (data.unusedBytes) node.unusedBytes = (node.unusedBytes || 0) + data.unusedBytes; + + // Only leaf nodes might have duplication data. + if (isLeaf && data.duplicatedNormalizedModuleName !== undefined) { + node.duplicatedNormalizedModuleName = data.duplicatedNormalizedModuleName; + } + }); + } + + // For every source file, apply the data to all components + // of the source path, creating nodes as necessary. + for (const [source, data] of Object.entries(sourcesData)) { + addAllNodesInSourcePath(source, data); + } + + /** + * Collapse nodes that have only one child. + * @param {Node} node + */ + function collapseAll(node) { + while (node.children && node.children.length === 1) { + node.name += '/' + node.children[0].name; + node.children = node.children[0].children; + } + + if (node.children) { + for (const child of node.children) { + collapseAll(child); + } + } + } + collapseAll(topNode); + + return topNode; + } + + /** + * Returns root node containers where the first level of nodes are script URLs. + * If a script has a source map, that node will be set by prepareTreemapNodes. + * @param {LH.Artifacts} artifacts + * @param {LH.Audit.Context} context + * @return {Promise} + */ + static async makeRootNodes(artifacts, context) { + /** @type {RootNodeContainer[]} */ + const rootNodeContainers = []; + + let inlineScriptLength = 0; + for (const scriptElement of artifacts.ScriptElements) { + // No src means script is inline. + // Combine these ScriptElements so that inline scripts show up as a single root node. + if (!scriptElement.src) { + inlineScriptLength += (scriptElement.content || '').length; + } + } + if (inlineScriptLength) { + const name = artifacts.URL.finalUrl; + rootNodeContainers.push({ + name, + node: { + name, + resourceBytes: inlineScriptLength, + }, + }); + } + + const bundles = await JsBundles.request(artifacts, context); + const duplicationByPath = await ModuleDuplication.request(artifacts, context); + + for (const scriptElement of artifacts.ScriptElements) { + if (!scriptElement.src) continue; + + const name = scriptElement.src; + const bundle = bundles.find(bundle => scriptElement.src === bundle.script.src); + const scriptCoverages = artifacts.JsUsage[scriptElement.src]; + if (!bundle || !scriptCoverages) { + // No bundle or coverage information, so simply make a single node + // detailing how big the script is. + + rootNodeContainers.push({ + name, + node: { + name, + resourceBytes: scriptElement.src.length, + }, + }); + continue; + } + + const unusedJavascriptSummary = await UnusedJavaScriptSummary.request( + {url: scriptElement.src, scriptCoverages, bundle}, context); + + /** @type {Node} */ + let node; + if (unusedJavascriptSummary.sourcesWastedBytes && !('errorMessage' in bundle.sizes)) { + // Create nodes for each module in a bundle. + + /** @type {Record} */ + const sourcesData = {}; + for (const source of Object.keys(bundle.sizes.files)) { + /** @type {SourceData} */ + const sourceData = { + resourceBytes: bundle.sizes.files[source], + unusedBytes: unusedJavascriptSummary.sourcesWastedBytes[source], + }; + + const key = ModuleDuplication.normalizeSource(source); + if (duplicationByPath.has(key)) sourceData.duplicatedNormalizedModuleName = key; + + sourcesData[source] = sourceData; + } + + node = this.prepareTreemapNodes(bundle.rawMap.sourceRoot || '', sourcesData); + } else { + // There was no source map for this script, so we can only produce a single node. + + node = { + name, + resourceBytes: unusedJavascriptSummary.totalBytes, + unusedBytes: unusedJavascriptSummary.wastedBytes, + }; + } + + rootNodeContainers.push({ + name, + node, + }); + } + + return rootNodeContainers; + } + + /** + * @param {LH.Artifacts} artifacts + * @param {LH.Audit.Context} context + * @return {Promise} + */ + static async audit(artifacts, context) { + const treemapData = await ScriptTreemapDataAudit.makeRootNodes(artifacts, context); + + // TODO: when out of experimental should make a new detail type. + /** @type {LH.Audit.Details.DebugData} */ + const details = { + type: 'debugdata', + treemapData, + }; + + return { + score: 1, + details, + }; + } +} + +module.exports = ScriptTreemapDataAudit; diff --git a/lighthouse-core/computed/module-duplication.js b/lighthouse-core/computed/module-duplication.js index 73b974868893..60bca2adbe9d 100644 --- a/lighthouse-core/computed/module-duplication.js +++ b/lighthouse-core/computed/module-duplication.js @@ -15,7 +15,7 @@ class ModuleDuplication { /** * @param {string} source */ - static _normalizeSource(source) { + static normalizeSource(source) { // Trim trailing question mark - b/c webpack. source = source.replace(/\?$/, ''); @@ -103,7 +103,7 @@ class ModuleDuplication { const sourceKey = (rawMap.sourceRoot || '') + rawMap.sources[i]; const sourceSize = sizes.files[sourceKey]; sourceDataArray.push({ - source: ModuleDuplication._normalizeSource(rawMap.sources[i]), + source: ModuleDuplication.normalizeSource(rawMap.sources[i]), resourceSize: sourceSize, }); } diff --git a/lighthouse-core/computed/resource-summary.js b/lighthouse-core/computed/resource-summary.js index 4ee005dd8f17..0071d086bb12 100644 --- a/lighthouse-core/computed/resource-summary.js +++ b/lighthouse-core/computed/resource-summary.js @@ -40,6 +40,7 @@ class ResourceSummary { * @return {Record} */ static summarize(networkRecords, mainResourceURL, context) { + /** @type {Record} */ const resourceSummary = { 'stylesheet': {count: 0, resourceSize: 0, transferSize: 0}, 'image': {count: 0, resourceSize: 0, transferSize: 0}, diff --git a/lighthouse-core/config/experimental-config.js b/lighthouse-core/config/experimental-config.js index ecc9cf0f9f57..2561a64f37f0 100644 --- a/lighthouse-core/config/experimental-config.js +++ b/lighthouse-core/config/experimental-config.js @@ -17,6 +17,7 @@ const config = { 'autocomplete', 'full-page-screenshot', 'large-javascript-libraries', + 'script-treemap-data', ], passes: [{ passName: 'defaultPass', diff --git a/lighthouse-core/test/audits/__snapshots__/script-treemap-data-test.js.snap b/lighthouse-core/test/audits/__snapshots__/script-treemap-data-test.js.snap new file mode 100644 index 000000000000..1897a538b0e3 --- /dev/null +++ b/lighthouse-core/test/audits/__snapshots__/script-treemap-data-test.js.snap @@ -0,0 +1,590 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`ScriptTreemapData audit squoosh fixture has root nodes 3`] = ` +Array [ + Object { + "name": "https://squoosh.app/main-app.js", + "node": Object { + "children": Array [ + Object { + "children": Array [ + Object { + "children": Array [ + Object { + "name": "util.ts", + "resourceBytes": 4043, + "unusedBytes": 500, + }, + Object { + "name": "icons.tsx", + "resourceBytes": 2531, + "unusedBytes": 24, + }, + Object { + "name": "clean-modify.ts", + "resourceBytes": 331, + }, + ], + "name": "lib", + "resourceBytes": 6905, + "unusedBytes": 524, + }, + Object { + "children": Array [ + Object { + "children": Array [ + Object { + "name": "style.scss", + "resourceBytes": 410, + }, + Object { + "name": "index.tsx", + "resourceBytes": 2176, + "unusedBytes": 37, + }, + ], + "name": "Options", + "resourceBytes": 2586, + "unusedBytes": 37, + }, + Object { + "children": Array [ + Object { + "children": Array [ + Object { + "children": Array [ + Object { + "name": "styles.css", + "resourceBytes": 75, + }, + Object { + "name": "index.ts", + "resourceBytes": 2088, + "unusedBytes": 42, + }, + ], + "name": "TwoUp", + "resourceBytes": 2163, + "unusedBytes": 42, + }, + Object { + "children": undefined, + "name": "PinchZoom/index.ts", + "resourceBytes": 3653, + "unusedBytes": 73, + }, + ], + "name": "custom-els", + "resourceBytes": 5816, + "unusedBytes": 115, + }, + Object { + "name": "style.scss", + "resourceBytes": 447, + }, + Object { + "name": "index.tsx", + "resourceBytes": 5199, + "unusedBytes": 73, + }, + ], + "name": "Output", + "resourceBytes": 11462, + "unusedBytes": 188, + }, + Object { + "children": Array [ + Object { + "name": "style.scss", + "resourceBytes": 780, + }, + Object { + "name": "FileSize.tsx", + "resourceBytes": 445, + "unusedBytes": 21, + }, + Object { + "name": "index.tsx", + "resourceBytes": 1538, + "unusedBytes": 19, + }, + ], + "name": "results", + "resourceBytes": 2763, + "unusedBytes": 40, + }, + Object { + "children": Array [ + Object { + "name": "style.scss", + "resourceBytes": 132, + }, + Object { + "children": Array [ + Object { + "name": "styles.css", + "resourceBytes": 105, + }, + Object { + "name": "index.ts", + "resourceBytes": 3461, + "unusedBytes": 28, + }, + ], + "name": "custom-els/MultiPanel", + "resourceBytes": 3566, + "unusedBytes": 28, + }, + Object { + "name": "result-cache.ts", + "resourceBytes": 611, + "unusedBytes": 20, + }, + Object { + "name": "index.tsx", + "resourceBytes": 8782, + "unusedBytes": 28, + }, + ], + "name": "compress", + "resourceBytes": 13091, + "unusedBytes": 76, + }, + Object { + "children": Array [ + Object { + "name": "style.scss", + "resourceBytes": 200, + }, + Object { + "name": "index.tsx", + "resourceBytes": 566, + "unusedBytes": 9, + }, + ], + "name": "range", + "resourceBytes": 766, + "unusedBytes": 9, + }, + Object { + "children": Array [ + Object { + "name": "style.scss", + "resourceBytes": 106, + }, + Object { + "name": "index.tsx", + "resourceBytes": 247, + "unusedBytes": 1, + }, + ], + "name": "checkbox", + "resourceBytes": 353, + "unusedBytes": 1, + }, + Object { + "children": Array [ + Object { + "name": "style.scss", + "resourceBytes": 66, + }, + Object { + "name": "index.tsx", + "resourceBytes": 901, + "unusedBytes": 21, + }, + ], + "name": "expander", + "resourceBytes": 967, + "unusedBytes": 21, + }, + Object { + "children": Array [ + Object { + "name": "style.scss", + "resourceBytes": 103, + }, + Object { + "name": "index.tsx", + "resourceBytes": 291, + "unusedBytes": 21, + }, + ], + "name": "select", + "resourceBytes": 394, + "unusedBytes": 21, + }, + ], + "name": "components", + "resourceBytes": 32382, + "unusedBytes": 393, + }, + Object { + "children": Array [ + Object { + "children": Array [ + Object { + "name": "util.ts", + "resourceBytes": 159, + }, + Object { + "name": "quality-option.tsx", + "resourceBytes": 398, + "unusedBytes": 19, + }, + ], + "name": "generic", + "resourceBytes": 557, + "unusedBytes": 19, + }, + Object { + "children": Array [ + Object { + "name": "encoder-meta.ts", + "resourceBytes": 268, + "unusedBytes": 44, + }, + Object { + "name": "encoder.tsx", + "resourceBytes": 101, + "unusedBytes": 22, + }, + ], + "name": "browser-png", + "resourceBytes": 369, + "unusedBytes": 66, + }, + Object { + "children": Array [ + Object { + "name": "encoder-meta.ts", + "resourceBytes": 282, + "unusedBytes": 44, + }, + Object { + "name": "encoder.ts", + "resourceBytes": 115, + "unusedBytes": 22, + }, + Object { + "name": "options.ts", + "resourceBytes": 35, + "unusedBytes": 35, + }, + ], + "name": "browser-jpeg", + "resourceBytes": 432, + "unusedBytes": 101, + }, + Object { + "children": Array [ + Object { + "name": "encoder-meta.ts", + "resourceBytes": 358, + "unusedBytes": 44, + }, + Object { + "name": "encoder.ts", + "resourceBytes": 115, + "unusedBytes": 22, + }, + Object { + "name": "options.ts", + "resourceBytes": 34, + "unusedBytes": 34, + }, + ], + "name": "browser-webp", + "resourceBytes": 507, + "unusedBytes": 100, + }, + Object { + "children": Array [ + Object { + "name": "encoder-meta.ts", + "resourceBytes": 343, + "unusedBytes": 44, + }, + Object { + "name": "encoder.ts", + "resourceBytes": 101, + "unusedBytes": 22, + }, + ], + "name": "browser-gif", + "resourceBytes": 444, + "unusedBytes": 66, + }, + Object { + "children": Array [ + Object { + "name": "encoder-meta.ts", + "resourceBytes": 347, + "unusedBytes": 44, + }, + Object { + "name": "encoder.ts", + "resourceBytes": 101, + "unusedBytes": 22, + }, + ], + "name": "browser-tiff", + "resourceBytes": 448, + "unusedBytes": 66, + }, + Object { + "children": Array [ + Object { + "name": "encoder-meta.ts", + "resourceBytes": 349, + "unusedBytes": 44, + }, + Object { + "name": "encoder.ts", + "resourceBytes": 101, + "unusedBytes": 22, + }, + ], + "name": "browser-jp2", + "resourceBytes": 450, + "unusedBytes": 66, + }, + Object { + "children": Array [ + Object { + "name": "encoder-meta.ts", + "resourceBytes": 343, + "unusedBytes": 44, + }, + Object { + "name": "encoder.ts", + "resourceBytes": 101, + "unusedBytes": 22, + }, + ], + "name": "browser-bmp", + "resourceBytes": 444, + "unusedBytes": 66, + }, + Object { + "children": Array [ + Object { + "name": "encoder-meta.ts", + "resourceBytes": 349, + "unusedBytes": 44, + }, + Object { + "name": "encoder.ts", + "resourceBytes": 101, + "unusedBytes": 22, + }, + ], + "name": "browser-pdf", + "resourceBytes": 450, + "unusedBytes": 66, + }, + Object { + "name": "tiny.webp", + "resourceBytes": 89, + }, + Object { + "name": "processor.ts", + "resourceBytes": 2380, + "unusedBytes": 22, + }, + Object { + "children": undefined, + "name": "processor-worker/index.ts", + "resourceBytes": 50, + }, + Object { + "children": Array [ + Object { + "name": "util.ts", + "resourceBytes": 134, + }, + Object { + "name": "processor-sync.ts", + "resourceBytes": 462, + "unusedBytes": 13, + }, + Object { + "name": "processor-meta.ts", + "resourceBytes": 225, + }, + Object { + "name": "options.tsx", + "resourceBytes": 3970, + "unusedBytes": 53, + }, + ], + "name": "resize", + "resourceBytes": 4791, + "unusedBytes": 66, + }, + Object { + "children": Array [ + Object { + "name": "encoder-meta.ts", + "resourceBytes": 436, + "unusedBytes": 3, + }, + Object { + "name": "options.tsx", + "resourceBytes": 4416, + "unusedBytes": 21, + }, + ], + "name": "mozjpeg", + "resourceBytes": 4852, + "unusedBytes": 24, + }, + Object { + "children": Array [ + Object { + "name": "options.tsx", + "resourceBytes": 366, + "unusedBytes": 21, + }, + Object { + "name": "encoder-meta.ts", + "resourceBytes": 59, + }, + ], + "name": "optipng", + "resourceBytes": 425, + "unusedBytes": 21, + }, + Object { + "children": Array [ + Object { + "name": "encoder-meta.ts", + "resourceBytes": 660, + "unusedBytes": 660, + }, + Object { + "name": "options.tsx", + "resourceBytes": 5114, + "unusedBytes": 93, + }, + ], + "name": "webp", + "resourceBytes": 5774, + "unusedBytes": 753, + }, + Object { + "children": Array [ + Object { + "name": "options.tsx", + "resourceBytes": 1052, + "unusedBytes": 44, + }, + Object { + "name": "processor-meta.ts", + "resourceBytes": 40, + }, + ], + "name": "imagequant", + "resourceBytes": 1092, + "unusedBytes": 44, + }, + Object { + "children": undefined, + "name": "identity/encoder-meta.ts", + "resourceBytes": 46, + }, + Object { + "name": "encoders.ts", + "resourceBytes": 336, + }, + Object { + "name": "preprocessors.ts", + "resourceBytes": 75, + }, + Object { + "name": "decoders.ts", + "resourceBytes": 206, + }, + Object { + "children": undefined, + "name": "rotate/processor-meta.ts", + "resourceBytes": 18, + "unusedBytes": 18, + }, + Object { + "name": "input-processors.ts", + "resourceBytes": 11, + "unusedBytes": 11, + }, + ], + "name": "codecs", + "resourceBytes": 24246, + "unusedBytes": 1575, + }, + Object { + "children": Array [ + Object { + "name": "styles.css", + "resourceBytes": 180, + }, + Object { + "name": "index.ts", + "resourceBytes": 2138, + "unusedBytes": 293, + }, + ], + "name": "custom-els/RangeInput", + "resourceBytes": 2318, + "unusedBytes": 293, + }, + ], + "name": "src", + "resourceBytes": 65851, + "unusedBytes": 2785, + }, + Object { + "children": Array [ + Object { + "children": undefined, + "name": "comlink/comlink.js", + "resourceBytes": 4117, + "unusedBytes": 256, + }, + Object { + "children": undefined, + "name": "pretty-bytes/index.js", + "resourceBytes": 635, + "unusedBytes": 11, + }, + Object { + "children": undefined, + "name": "pointer-tracker/dist/PointerTracker.mjs", + "resourceBytes": 2672, + "unusedBytes": 13, + }, + Object { + "children": undefined, + "name": "linkstate/dist/linkstate.es.js", + "resourceBytes": 412, + "unusedBytes": 1, + }, + ], + "name": "node_modules", + "resourceBytes": 7836, + "unusedBytes": 281, + }, + ], + "name": "/webpack:/.", + "resourceBytes": 73687, + "unusedBytes": 3066, + }, + }, + Object { + "name": "https://sqoosh.app/no-map-or-usage.js", + "node": Object { + "name": "https://sqoosh.app/no-map-or-usage.js", + "resourceBytes": 37, + }, + }, +] +`; diff --git a/lighthouse-core/test/audits/byte-efficiency/duplicated-javascript-test.js b/lighthouse-core/test/audits/byte-efficiency/duplicated-javascript-test.js index 942aeee1c323..03aaf9d47827 100644 --- a/lighthouse-core/test/audits/byte-efficiency/duplicated-javascript-test.js +++ b/lighthouse-core/test/audits/byte-efficiency/duplicated-javascript-test.js @@ -7,24 +7,15 @@ /* eslint-env jest */ -const fs = require('fs'); const DuplicatedJavascript = require('../../../audits/byte-efficiency/duplicated-javascript.js'); const trace = require('../../fixtures/traces/lcp-m78.json'); const devtoolsLog = require('../../fixtures/traces/lcp-m78.devtools.log.json'); - -function load(name) { - const mapJson = fs.readFileSync( - `${__dirname}/../../fixtures/source-maps/${name}.js.map`, - 'utf-8' - ); - const content = fs.readFileSync(`${__dirname}/../../fixtures/source-maps/${name}.js`, 'utf-8'); - return {map: JSON.parse(mapJson), content}; -} +const {loadSourceMapFixture} = require('../../test-utils.js'); describe('DuplicatedJavascript computed artifact', () => { it('works (simple)', async () => { const context = {computedCache: new Map(), options: {ignoreThresholdInBytes: 200}}; - const {map, content} = load('foo.min'); + const {map, content} = loadSourceMapFixture('foo.min'); const artifacts = { URL: {finalUrl: 'https://example.com'}, SourceMaps: [ @@ -49,8 +40,8 @@ describe('DuplicatedJavascript computed artifact', () => { it('works (complex)', async () => { const context = {computedCache: new Map(), options: {ignoreThresholdInBytes: 200}}; - const bundleData1 = load('coursehero-bundle-1'); - const bundleData2 = load('coursehero-bundle-2'); + const bundleData1 = loadSourceMapFixture('coursehero-bundle-1'); + const bundleData2 = loadSourceMapFixture('coursehero-bundle-2'); const artifacts = { URL: {finalUrl: 'https://example.com'}, SourceMaps: [ @@ -324,8 +315,8 @@ describe('DuplicatedJavascript computed artifact', () => { it('.audit', async () => { // Use a real trace fixture, but the bundle stuff. - const bundleData1 = load('coursehero-bundle-1'); - const bundleData2 = load('coursehero-bundle-2'); + const bundleData1 = loadSourceMapFixture('coursehero-bundle-1'); + const bundleData2 = loadSourceMapFixture('coursehero-bundle-2'); const artifacts = { URL: {finalUrl: 'https://www.paulirish.com'}, devtoolsLogs: { diff --git a/lighthouse-core/test/audits/byte-efficiency/unused-javascript-test.js b/lighthouse-core/test/audits/byte-efficiency/unused-javascript-test.js index 94902a755634..1992346ac9f6 100644 --- a/lighthouse-core/test/audits/byte-efficiency/unused-javascript-test.js +++ b/lighthouse-core/test/audits/byte-efficiency/unused-javascript-test.js @@ -6,43 +6,26 @@ 'use strict'; const assert = require('assert').strict; -const fs = require('fs'); const UnusedJavaScript = require('../../../audits/byte-efficiency/unused-javascript.js'); const networkRecordsToDevtoolsLog = require('../../network-records-to-devtools-log.js'); - -function load(name) { - const dir = `${__dirname}/../../fixtures/source-maps`; - const mapJson = fs.readFileSync(`${dir}/${name}.js.map`, 'utf-8'); - const content = fs.readFileSync(`${dir}/${name}.js`, 'utf-8'); - const usageJson = fs.readFileSync(`${dir}/${name}.usage.json`, 'utf-8'); - const exportedUsage = JSON.parse(usageJson); - - // Usage is exported from DevTools, which simplifies the real format of the - // usage protocol. - const usage = { - url: exportedUsage.url, - functions: [ - { - ranges: exportedUsage.ranges.map((range, i) => { - return { - startOffset: range.start, - endOffset: range.end, - count: i % 2 === 0 ? 0 : 1, - }; - }), - }, - ], - }; - - return {map: JSON.parse(mapJson), content, usage}; -} +const {loadSourceMapAndUsageFixture} = require('../../test-utils.js'); /* eslint-env jest */ +/** + * @param {string} url + * @param {number} transferSize + * @param {LH.Crdp.Network.ResourceType} resourceType + */ function generateRecord(url, transferSize, resourceType) { return {url, transferSize, resourceType}; } +/** + * @param {string} url + * @param {Array<[number, number, number]>} ranges + * @return {Crdp.Profiler.ScriptCoverage} + */ function generateUsage(url, ranges) { const functions = ranges.map(range => { return { @@ -120,7 +103,7 @@ describe('UnusedJavaScript audit', () => { bundleSourceUnusedThreshold: 100, }, }; - const {map, content, usage} = load('squoosh'); + const {map, content, usage} = loadSourceMapAndUsageFixture('squoosh'); const url = 'https://squoosh.app/main-app.js'; const networkRecords = [generateRecord(url, content.length, 'Script')]; const artifacts = { diff --git a/lighthouse-core/test/audits/script-treemap-data-test.js b/lighthouse-core/test/audits/script-treemap-data-test.js new file mode 100644 index 000000000000..5a7474d5b094 --- /dev/null +++ b/lighthouse-core/test/audits/script-treemap-data-test.js @@ -0,0 +1,301 @@ +/** + * @license Copyright 2020 The Lighthouse Authors. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/* eslint-env jest */ + +const ScriptTreemapData_ = require('../../audits/script-treemap-data.js'); +const networkRecordsToDevtoolsLog = require('../network-records-to-devtools-log.js'); +const {loadSourceMapAndUsageFixture, makeParamsOptional} = require('../test-utils.js'); + +const ScriptTreemapData = { + audit: makeParamsOptional(ScriptTreemapData_.audit), + prepareTreemapNodes: makeParamsOptional(ScriptTreemapData_.prepareTreemapNodes), +}; + +/** + * @param {string} url + * @param {number} resourceSize + * @param {LH.Crdp.Network.ResourceType} resourceType + */ +function generateRecord(url, resourceSize, resourceType) { + return {url, resourceSize, resourceType}; +} + +describe('ScriptTreemapData audit', () => { + describe('squoosh fixture', () => { + /** @type {import('../../audits/script-treemap-data.js').TreemapData} */ + let treemapData; + beforeAll(async () => { + const context = {computedCache: new Map()}; + const {map, content, usage} = loadSourceMapAndUsageFixture('squoosh'); + const mainUrl = 'https://squoosh.app'; + const scriptUrl = 'https://squoosh.app/main-app.js'; + const networkRecords = [generateRecord(scriptUrl, content.length, 'Script')]; + + // Add a script with no source map or usage. + const noSourceMapScript = {src: 'https://sqoosh.app/no-map-or-usage.js', content: '// hi'}; + networkRecords.push( + generateRecord(noSourceMapScript.src, noSourceMapScript.content.length, 'Script') + ); + + const artifacts = { + URL: {requestedUrl: mainUrl, finalUrl: mainUrl}, + JsUsage: {[usage.url]: [usage]}, + devtoolsLogs: {defaultPass: networkRecordsToDevtoolsLog(networkRecords)}, + SourceMaps: [{scriptUrl: scriptUrl, map}], + ScriptElements: [{src: scriptUrl, content}, noSourceMapScript], + }; + const results = await ScriptTreemapData.audit(artifacts, context); + + // @ts-expect-error: Debug data. + treemapData = results.details.treemapData; + }); + + it('has root nodes', () => { + expect(treemapData.find(s => s.name === 'https://sqoosh.app/no-map-or-usage.js')) + .toMatchInlineSnapshot(` + Object { + "name": "https://sqoosh.app/no-map-or-usage.js", + "node": Object { + "name": "https://sqoosh.app/no-map-or-usage.js", + "resourceBytes": 37, + }, + } + `); + + expect(JSON.stringify(treemapData).length).toMatchInlineSnapshot(`6621`); + expect(treemapData).toMatchSnapshot(); + }); + }); + + describe('.prepareTreemapNodes', () => { + it('uses node data when available', () => { + const rootNode = ScriptTreemapData.prepareTreemapNodes('', { + 'a.js': {resourceBytes: 100}, + 'b.js': {resourceBytes: 100, duplicatedNormalizedModuleName: 'blah'}, + 'c.js': {resourceBytes: 100, unusedBytes: 50}, + }); + expect(rootNode).toMatchObject( + { + children: [ + { + name: 'a.js', + resourceBytes: 100, + }, + { + duplicatedNormalizedModuleName: 'blah', + name: 'b.js', + resourceBytes: 100, + }, + { + name: 'c.js', + resourceBytes: 100, + unusedBytes: 50, + }, + ], + name: '', + resourceBytes: 300, + unusedBytes: 50, + } + ); + }); + + it('creates directory node when multiple leaf nodes', () => { + const rootNode = ScriptTreemapData.prepareTreemapNodes('', { + 'folder/a.js': {resourceBytes: 100}, + 'folder/b.js': {resourceBytes: 100}, + }); + expect(rootNode).toMatchObject( + { + children: [ + { + name: 'a.js', + resourceBytes: 100, + }, + { + name: 'b.js', + resourceBytes: 100, + }, + ], + name: '/folder', + resourceBytes: 200, + } + ); + }); + + it('flattens directory node when single leaf nodes', () => { + const rootNode = ScriptTreemapData.prepareTreemapNodes('', { + 'root/folder1/a.js': {resourceBytes: 100}, + 'root/folder2/b.js': {resourceBytes: 100}, + }); + expect(rootNode).toMatchObject( + { + children: [ + { + children: undefined, + name: 'folder1/a.js', + resourceBytes: 100, + }, + { + children: undefined, + name: 'folder2/b.js', + resourceBytes: 100, + }, + ], + name: '/root', + resourceBytes: 200, + } + ); + }); + + it('source root replaces matching prefixes', () => { + const sourcesData = { + 'some/prefix/main.js': {resourceBytes: 100, unusedBytes: 50}, + 'not/some/prefix/a.js': {resourceBytes: 101, unusedBytes: 51}, + }; + const rootNode = ScriptTreemapData.prepareTreemapNodes('some/prefix', sourcesData); + expect(rootNode).toMatchObject( + { + children: [ + { + children: undefined, + name: '/main.js', + resourceBytes: 100, + unusedBytes: 50, + }, + { + children: undefined, + name: 'not/a.js', + resourceBytes: 101, + unusedBytes: 51, + }, + ], + name: 'some/prefix', + resourceBytes: 201, + unusedBytes: 101, + } + ); + + expect(rootNode.name).toBe('some/prefix'); + expect(rootNode.resourceBytes).toBe(201); + expect(rootNode.unusedBytes).toBe(101); + + const children = rootNode.children || []; + expect(children[0].name).toBe('/main.js'); + expect(children[1].name).toBe('not/a.js'); + }); + + it('nodes have unusedBytes data', () => { + const sourcesData = { + 'lib/folder/a.js': {resourceBytes: 100, unusedBytes: 50}, + 'lib/folder/b.js': {resourceBytes: 101}, + 'lib/c.js': {resourceBytes: 100, unusedBytes: 25}, + }; + const rootNode = ScriptTreemapData.prepareTreemapNodes('', sourcesData); + expect(rootNode).toMatchObject( + { + children: [ + { + children: [ + { + name: 'a.js', + resourceBytes: 100, + unusedBytes: 50, + }, + { + name: 'b.js', + resourceBytes: 101, + }, + ], + name: 'folder', + resourceBytes: 201, + unusedBytes: 50, + }, + { + name: 'c.js', + resourceBytes: 100, + unusedBytes: 25, + }, + ], + name: '/lib', + resourceBytes: 301, + unusedBytes: 75, + } + ); + }); + + it('nodes have duplicates data', () => { + const sourcesData = { + /* eslint-disable max-len */ + 'lib/folder/a.js': {resourceBytes: 100, unusedBytes: 50}, + 'lib/node_modules/dep/a.js': {resourceBytes: 101, duplicatedNormalizedModuleName: 'dep/a.js'}, + 'node_modules/dep/a.js': {resourceBytes: 100, unusedBytes: 25, duplicatedNormalizedModuleName: 'dep/a.js'}, + 'lib/node_modules/dep/b.js': {resourceBytes: 101, duplicatedNormalizedModuleName: 'dep/b.js'}, + 'node_modules/dep/b.js': {resourceBytes: 100, unusedBytes: 25, duplicatedNormalizedModuleName: 'dep/b.js'}, + /* eslint-enable max-len */ + }; + const rootNode = ScriptTreemapData.prepareTreemapNodes('', sourcesData); + expect(rootNode).toMatchObject( + { + children: [ + { + children: [ + { + children: undefined, + name: 'folder/a.js', + resourceBytes: 100, + unusedBytes: 50, + }, + { + children: [ + { + duplicatedNormalizedModuleName: 'dep/a.js', + name: 'a.js', + resourceBytes: 101, + }, + { + duplicatedNormalizedModuleName: 'dep/b.js', + name: 'b.js', + resourceBytes: 101, + }, + ], + name: 'node_modules/dep', + resourceBytes: 202, + }, + ], + name: 'lib', + resourceBytes: 302, + unusedBytes: 50, + }, + { + children: [ + { + duplicatedNormalizedModuleName: 'dep/a.js', + name: 'a.js', + resourceBytes: 100, + unusedBytes: 25, + }, + { + duplicatedNormalizedModuleName: 'dep/b.js', + name: 'b.js', + resourceBytes: 100, + unusedBytes: 25, + }, + ], + name: 'node_modules/dep', + resourceBytes: 200, + unusedBytes: 50, + }, + ], + name: '', + resourceBytes: 502, + unusedBytes: 100, + } + ); + }); + }); +}); diff --git a/lighthouse-core/test/audits/valid-source-maps-test.js b/lighthouse-core/test/audits/valid-source-maps-test.js index fa72f3e840c3..c4f969488243 100644 --- a/lighthouse-core/test/audits/valid-source-maps-test.js +++ b/lighthouse-core/test/audits/valid-source-maps-test.js @@ -6,16 +6,11 @@ 'use strict'; /* eslint-env jest */ -function load(name) { - const mapJson = fs.readFileSync(`${__dirname}/../fixtures/source-maps/${name}.js.map`, 'utf-8'); - const content = fs.readFileSync(`${__dirname}/../fixtures/source-maps/${name}.js`, 'utf-8'); - return {map: JSON.parse(mapJson), content}; -} +const {loadSourceMapFixture} = require('../test-utils.js'); const ValidSourceMaps = require('../../audits/valid-source-maps.js'); -const fs = require('fs'); -const largeBundle = load('coursehero-bundle-1'); -const smallBundle = load('coursehero-bundle-2'); +const largeBundle = loadSourceMapFixture('coursehero-bundle-1'); +const smallBundle = loadSourceMapFixture('coursehero-bundle-2'); const LARGE_JS_BYTE_THRESHOLD = 500 * 1024; if (largeBundle.content.length < LARGE_JS_BYTE_THRESHOLD) { @@ -111,8 +106,8 @@ describe('Valid source maps audit', () => { }); it('discovers missing source map contents while passing', async () => { - const bundleNormal = load('squoosh'); - const bundleWithMissingContent = load('squoosh'); + const bundleNormal = loadSourceMapFixture('squoosh'); + const bundleWithMissingContent = loadSourceMapFixture('squoosh'); delete bundleWithMissingContent.map.sourcesContent[0]; const artifacts = { @@ -142,7 +137,7 @@ describe('Valid source maps audit', () => { }); it('discovers missing source map contents while failing', async () => { - const bundleWithMissingContent = load('squoosh'); + const bundleWithMissingContent = loadSourceMapFixture('squoosh'); delete bundleWithMissingContent.map.sourcesContent[0]; const artifacts = { diff --git a/lighthouse-core/test/computed/js-bundles-test.js b/lighthouse-core/test/computed/js-bundles-test.js index 7a68b0f48907..3e07b84c8bd3 100644 --- a/lighthouse-core/test/computed/js-bundles-test.js +++ b/lighthouse-core/test/computed/js-bundles-test.js @@ -6,14 +6,8 @@ 'use strict'; /* eslint-env jest */ -const fs = require('fs'); const JsBundles = require('../../computed/js-bundles.js'); - -function load(name) { - const mapJson = fs.readFileSync(`${__dirname}/../fixtures/source-maps/${name}.js.map`, 'utf-8'); - const content = fs.readFileSync(`${__dirname}/../fixtures/source-maps/${name}.js`, 'utf-8'); - return {map: JSON.parse(mapJson), content}; -} +const {loadSourceMapFixture} = require('../test-utils.js'); describe('JsBundles computed artifact', () => { it('collates script element and source map', async () => { @@ -34,7 +28,7 @@ describe('JsBundles computed artifact', () => { it('works (simple map)', async () => { // This map is from source-map-explorer. // https://github.com/danvk/source-map-explorer/tree/4b95f6e7dfe0058d791dcec2107fee43a1ebf02e/tests - const {map, content} = load('foo.min'); + const {map, content} = loadSourceMapFixture('foo.min'); const artifacts = { SourceMaps: [{scriptUrl: 'https://example.com/foo.min.js', map}], ScriptElements: [{src: 'https://example.com/foo.min.js', content}], @@ -80,7 +74,7 @@ describe('JsBundles computed artifact', () => { it('works (simple map) (null source)', async () => { // This map is from source-map-explorer. // https://github.com/danvk/source-map-explorer/tree/4b95f6e7dfe0058d791dcec2107fee43a1ebf02e/tests - const {map, content} = load('foo.min'); + const {map, content} = loadSourceMapFixture('foo.min'); map.sources[1] = null; const artifacts = { SourceMaps: [{scriptUrl: 'https://example.com/foo.min.js', map}], @@ -123,7 +117,7 @@ describe('JsBundles computed artifact', () => { }); it('works (complex map)', async () => { - const {map, content} = load('squoosh'); + const {map, content} = loadSourceMapFixture('squoosh'); const artifacts = { SourceMaps: [{scriptUrl: 'https://squoosh.app/main-app.js', map}], ScriptElements: [{src: 'https://squoosh.app/main-app.js', content}], @@ -240,7 +234,7 @@ describe('JsBundles computed artifact', () => { let content; beforeEach(() => { - data = load('foo.min'); + data = loadSourceMapFixture('foo.min'); map = data.map; content = data.content; }); diff --git a/lighthouse-core/test/computed/module-duplication-test.js b/lighthouse-core/test/computed/module-duplication-test.js index 0933d490e903..78310204b6b8 100644 --- a/lighthouse-core/test/computed/module-duplication-test.js +++ b/lighthouse-core/test/computed/module-duplication-test.js @@ -7,19 +7,13 @@ /* eslint-env jest */ -const fs = require('fs'); const ModuleDuplication = require('../../computed/module-duplication.js'); - -function load(name) { - const mapJson = fs.readFileSync(`${__dirname}/../fixtures/source-maps/${name}.js.map`, 'utf-8'); - const content = fs.readFileSync(`${__dirname}/../fixtures/source-maps/${name}.js`, 'utf-8'); - return {map: JSON.parse(mapJson), content}; -} +const {loadSourceMapFixture} = require('../test-utils.js'); describe('ModuleDuplication computed artifact', () => { it('works (simple)', async () => { const context = {computedCache: new Map()}; - const {map, content} = load('foo.min'); + const {map, content} = loadSourceMapFixture('foo.min'); const artifacts = { SourceMaps: [ {scriptUrl: 'https://example.com/foo1.min.js', map}, @@ -36,8 +30,8 @@ describe('ModuleDuplication computed artifact', () => { it('works (complex)', async () => { const context = {computedCache: new Map()}; - const bundleData1 = load('coursehero-bundle-1'); - const bundleData2 = load('coursehero-bundle-2'); + const bundleData1 = loadSourceMapFixture('coursehero-bundle-1'); + const bundleData2 = loadSourceMapFixture('coursehero-bundle-2'); const artifacts = { SourceMaps: [ {scriptUrl: 'https://example.com/coursehero-bundle-1.js', map: bundleData1.map}, @@ -215,7 +209,7 @@ describe('ModuleDuplication computed artifact', () => { `); }); - it('_normalizeSource', () => { + it('normalizeSource', () => { const testCases = [ ['test.js', 'test.js'], ['node_modules/othermodule.js', 'node_modules/othermodule.js'], @@ -227,7 +221,7 @@ describe('ModuleDuplication computed artifact', () => { ['webpack.js?', 'webpack.js'], ]; for (const [input, expected] of testCases) { - expect(ModuleDuplication._normalizeSource(input)).toBe(expected); + expect(ModuleDuplication.normalizeSource(input)).toBe(expected); } }); diff --git a/lighthouse-core/test/gather/gather-runner-test.js b/lighthouse-core/test/gather/gather-runner-test.js index 977047718c6d..21e546659075 100644 --- a/lighthouse-core/test/gather/gather-runner-test.js +++ b/lighthouse-core/test/gather/gather-runner-test.js @@ -18,18 +18,10 @@ const networkRecordsToDevtoolsLog = require('../network-records-to-devtools-log. const Driver = require('../../gather/driver.js'); const Connection = require('../../gather/connections/connection.js'); const {createMockSendCommandFn} = require('./mock-commands.js'); +const {makeParamsOptional} = require('../test-utils.js'); jest.mock('../../lib/stack-collector.js', () => () => Promise.resolve([])); -/** - * @template {unknown[]} TParams - * @template TReturn - * @param {(...args: TParams) => TReturn} fn - */ -function makeParamsOptional(fn) { - return /** @type {(...args: RecursivePartial) => TReturn} */ (fn); -} - const GatherRunner = { afterPass: makeParamsOptional(GatherRunner_.afterPass), beginRecording: makeParamsOptional(GatherRunner_.beginRecording), diff --git a/lighthouse-core/test/test-utils.js b/lighthouse-core/test/test-utils.js index d4c7f94f0e57..14c485a5b4b7 100644 --- a/lighthouse-core/test/test-utils.js +++ b/lighthouse-core/test/test-utils.js @@ -38,7 +38,7 @@ expect.extend({ // done for asymmetric matchers anyways. const thisObj = (this && this.utils) ? this : {isNot: false, promise: ''}; - + // @ts-expect-error return toBeCloseTo.call(thisObj, ...args); }, }); @@ -78,6 +78,69 @@ function getProtoRoundTrip() { }; } +/** + * @param {string} name + * @return {{map: LH.Artifacts.RawSourceMap, content: string}} + */ +function loadSourceMapFixture(name) { + const dir = `${__dirname}/fixtures/source-maps`; + const mapJson = fs.readFileSync(`${dir}/${name}.js.map`, 'utf-8'); + const content = fs.readFileSync(`${dir}/${name}.js`, 'utf-8'); + return { + map: JSON.parse(mapJson), + content, + }; +} + +/** + * @param {string} name + * @return {{map: LH.Artifacts.RawSourceMap, content: string, usage: LH.Crdp.Profiler.ScriptCoverage}} + */ +function loadSourceMapAndUsageFixture(name) { + const dir = `${__dirname}/fixtures/source-maps`; + const usagePath = `${dir}/${name}.usage.json`; + const usageJson = fs.readFileSync(usagePath, 'utf-8'); + + // Usage is exported from DevTools, which simplifies the real format of the + // usage protocol. + /** @type {{url: string, ranges: Array<{start: number, end: number, count: number}>}} */ + const exportedUsage = JSON.parse(usageJson); + const usage = { + scriptId: 'FakeId', // Not used. + url: exportedUsage.url, + functions: [ + { + functionName: 'FakeFunctionName', // Not used. + isBlockCoverage: false, // Not used. + ranges: exportedUsage.ranges.map((range, i) => { + return { + startOffset: range.start, + endOffset: range.end, + count: i % 2 === 0 ? 0 : 1, + }; + }), + }, + ], + }; + + return { + ...loadSourceMapFixture(name), + usage, + }; +} + +/** + * @template {unknown[]} TParams + * @template TReturn + * @param {(...args: TParams) => TReturn} fn + */ +function makeParamsOptional(fn) { + return /** @type {(...args: RecursivePartial) => TReturn} */ (fn); +} + module.exports = { getProtoRoundTrip, + loadSourceMapFixture, + loadSourceMapAndUsageFixture, + makeParamsOptional, }; diff --git a/tsconfig.json b/tsconfig.json index 4e52101808c1..7081582269c5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -33,5 +33,6 @@ "lighthouse-core/test/scripts/lantern/constants-test.js", "lighthouse-core/test/gather/driver-test.js", "lighthouse-core/test/gather/gather-runner-test.js", + "lighthouse-core/test/audits/script-treemap-data-test.js" ], }