diff --git a/core/computed/page-dependency-graph.js b/core/computed/page-dependency-graph.js index dde298025322..fef93f5371e7 100644 --- a/core/computed/page-dependency-graph.js +++ b/core/computed/page-dependency-graph.js @@ -5,463 +5,15 @@ */ import {makeComputedArtifact} from './computed-artifact.js'; -import {NetworkNode} from '../lib/lantern/network-node.js'; -import {CPUNode} from '../lib/lantern/cpu-node.js'; -import {TraceProcessor} from '../lib/tracehouse/trace-processor.js'; +import {PageDependencyGraph as LanternPageDependencyGraph} from '../lib/lantern/page-dependency-graph.js'; import {NetworkRequest} from '../lib/network-request.js'; import {ProcessedTrace} from './processed-trace.js'; import {NetworkRecords} from './network-records.js'; -import {NetworkAnalyzer} from '../lib/lantern/simulator/network-analyzer.js'; import {DocumentUrls} from './document-urls.js'; /** @typedef {import('../lib/lantern/base-node.js').Node} Node */ -/** @typedef {Omit} URLArtifact */ - -/** - * @typedef {Object} NetworkNodeOutput - * @property {Array>} nodes - * @property {Map>} idToNodeMap - * @property {Map>>} urlToNodeMap - * @property {Map|null>} frameIdToNodeMap - */ - -// Shorter tasks have negligible impact on simulation results. -const SIGNIFICANT_DUR_THRESHOLD_MS = 10; - -// TODO: video files tend to be enormous and throw off all graph traversals, move this ignore -// into estimation logic when we use the dependency graph for other purposes. -const IGNORED_MIME_TYPES_REGEX = /^video/; class PageDependencyGraph { - /** - * @param {LH.Artifacts.NetworkRequest} record - * @return {Array} - */ - static getNetworkInitiators(record) { - if (!record.initiator) return []; - if (record.initiator.url) return [record.initiator.url]; - if (record.initiator.type === 'script') { - // Script initiators have the stack of callFrames from all functions that led to this request. - // If async stacks are enabled, then the stack will also have the parent functions that asynchronously - // led to this request chained in the `parent` property. - /** @type {Set} */ - const scriptURLs = new Set(); - let stack = record.initiator.stack; - while (stack) { - const callFrames = stack.callFrames || []; - for (const frame of callFrames) { - if (frame.url) scriptURLs.add(frame.url); - } - - stack = stack.parent; - } - - return Array.from(scriptURLs); - } - - return []; - } - - /** - * @param {Array} networkRecords - * @return {NetworkNodeOutput} - */ - static getNetworkNodeOutput(networkRecords) { - /** @type {Array} */ - const nodes = []; - /** @type {Map} */ - const idToNodeMap = new Map(); - /** @type {Map>} */ - const urlToNodeMap = new Map(); - /** @type {Map} */ - const frameIdToNodeMap = new Map(); - - networkRecords.forEach(record => { - if (IGNORED_MIME_TYPES_REGEX.test(record.mimeType)) return; - if (record.sessionTargetType === 'worker') return; - - // Network record requestIds can be duplicated for an unknown reason - // Suffix all subsequent records with `:duplicate` until it's unique - // NOTE: This should never happen with modern NetworkRequest library, but old fixtures - // might still have this issue. - while (idToNodeMap.has(record.requestId)) { - record.requestId += ':duplicate'; - } - - const node = new NetworkNode(NetworkRequest.asLanternNetworkRequest(record)); - nodes.push(node); - - const urlList = urlToNodeMap.get(record.url) || []; - urlList.push(node); - - idToNodeMap.set(record.requestId, node); - urlToNodeMap.set(record.url, urlList); - - // If the request was for the root document of an iframe, save an entry in our - // map so we can link up the task `args.data.frame` dependencies later in graph creation. - if (record.frameId && - record.resourceType === NetworkRequest.TYPES.Document && - record.documentURL === record.url) { - // If there's ever any ambiguity, permanently set the value to `false` to avoid loops in the graph. - const value = frameIdToNodeMap.has(record.frameId) ? null : node; - frameIdToNodeMap.set(record.frameId, value); - } - }); - - return {nodes, idToNodeMap, urlToNodeMap, frameIdToNodeMap}; - } - - /** - * @param {LH.Artifacts.ProcessedTrace} processedTrace - * @return {Array} - */ - static getCPUNodes({mainThreadEvents}) { - /** @type {Array} */ - const nodes = []; - let i = 0; - - TraceProcessor.assertHasToplevelEvents(mainThreadEvents); - - while (i < mainThreadEvents.length) { - const evt = mainThreadEvents[i]; - i++; - - // Skip all trace events that aren't schedulable tasks with sizable duration - if (!TraceProcessor.isScheduleableTask(evt) || !evt.dur) { - continue; - } - - // Capture all events that occurred within the task - /** @type {Array} */ - const children = []; - for ( - const endTime = evt.ts + evt.dur; - i < mainThreadEvents.length && mainThreadEvents[i].ts < endTime; - i++ - ) { - children.push(mainThreadEvents[i]); - } - - nodes.push(new CPUNode(evt, children)); - } - - return nodes; - } - - /** - * @param {NetworkNode} rootNode - * @param {NetworkNodeOutput} networkNodeOutput - */ - static linkNetworkNodes(rootNode, networkNodeOutput) { - networkNodeOutput.nodes.forEach(node => { - const directInitiatorRequest = node.record.initiatorRequest || rootNode.record; - const directInitiatorNode = - networkNodeOutput.idToNodeMap.get(directInitiatorRequest.requestId) || rootNode; - const canDependOnInitiator = - !directInitiatorNode.isDependentOn(node) && - node.canDependOn(directInitiatorNode); - const initiators = PageDependencyGraph.getNetworkInitiators(node.record); - if (initiators.length) { - initiators.forEach(initiator => { - const parentCandidates = networkNodeOutput.urlToNodeMap.get(initiator) || []; - // Only add the edge if the parent is unambiguous with valid timing and isn't circular. - if (parentCandidates.length === 1 && - parentCandidates[0].startTime <= node.startTime && - !parentCandidates[0].isDependentOn(node)) { - node.addDependency(parentCandidates[0]); - } else if (canDependOnInitiator) { - directInitiatorNode.addDependent(node); - } - }); - } else if (canDependOnInitiator) { - directInitiatorNode.addDependent(node); - } - - // Make sure the nodes are attached to the graph if the initiator information was invalid. - if (node !== rootNode && node.getDependencies().length === 0 && node.canDependOn(rootNode)) { - node.addDependency(rootNode); - } - - if (!node.record.redirects) return; - - const redirects = [...node.record.redirects, node.record]; - for (let i = 1; i < redirects.length; i++) { - const redirectNode = networkNodeOutput.idToNodeMap.get(redirects[i - 1].requestId); - const actualNode = networkNodeOutput.idToNodeMap.get(redirects[i].requestId); - if (actualNode && redirectNode) { - actualNode.addDependency(redirectNode); - } - } - }); - } - - /** - * @param {Node} rootNode - * @param {NetworkNodeOutput} networkNodeOutput - * @param {Array} cpuNodes - */ - static linkCPUNodes(rootNode, networkNodeOutput, cpuNodes) { - /** @type {Set} */ - const linkableResourceTypes = new Set([ - NetworkRequest.TYPES.XHR, NetworkRequest.TYPES.Fetch, NetworkRequest.TYPES.Script, - ]); - - /** @param {CPUNode} cpuNode @param {string} reqId */ - function addDependentNetworkRequest(cpuNode, reqId) { - const networkNode = networkNodeOutput.idToNodeMap.get(reqId); - if (!networkNode || - // Ignore all network nodes that started before this CPU task started - // A network request that started earlier could not possibly have been started by this task - networkNode.startTime <= cpuNode.startTime) return; - const {record} = networkNode; - const resourceType = record.resourceType || - record.redirectDestination?.resourceType; - if (!linkableResourceTypes.has(resourceType)) { - // We only link some resources to CPU nodes because we observe LCP simulation - // regressions when including images, etc. - return; - } - cpuNode.addDependent(networkNode); - } - - /** - * If the node has an associated frameId, then create a dependency on the root document request - * for the frame. The task obviously couldn't have started before the frame was even downloaded. - * - * @param {CPUNode} cpuNode - * @param {string|undefined} frameId - */ - function addDependencyOnFrame(cpuNode, frameId) { - if (!frameId) return; - const networkNode = networkNodeOutput.frameIdToNodeMap.get(frameId); - if (!networkNode) return; - // Ignore all network nodes that started after this CPU task started - // A network request that started after could not possibly be required this task - if (networkNode.startTime >= cpuNode.startTime) return; - cpuNode.addDependency(networkNode); - } - - /** @param {CPUNode} cpuNode @param {string} url */ - function addDependencyOnUrl(cpuNode, url) { - if (!url) return; - // Allow network requests that end up to 100ms before the task started - // Some script evaluations can start before the script finishes downloading - const minimumAllowableTimeSinceNetworkNodeEnd = -100 * 1000; - const candidates = networkNodeOutput.urlToNodeMap.get(url) || []; - - let minCandidate = null; - let minDistance = Infinity; - // Find the closest request that finished before this CPU task started - for (const candidate of candidates) { - // Explicitly ignore all requests that started after this CPU node - // A network request that started after this task started cannot possibly be a dependency - if (cpuNode.startTime <= candidate.startTime) return; - - const distance = cpuNode.startTime - candidate.endTime; - if (distance >= minimumAllowableTimeSinceNetworkNodeEnd && distance < minDistance) { - minCandidate = candidate; - minDistance = distance; - } - } - - if (!minCandidate) return; - cpuNode.addDependency(minCandidate); - } - - /** @type {Map} */ - const timers = new Map(); - for (const node of cpuNodes) { - for (const evt of node.childEvents) { - if (!evt.args.data) continue; - - const argsUrl = evt.args.data.url; - const stackTraceUrls = (evt.args.data.stackTrace || []).map(l => l.url).filter(Boolean); - - switch (evt.name) { - case 'TimerInstall': - // @ts-expect-error - 'TimerInstall' event means timerId exists. - timers.set(evt.args.data.timerId, node); - stackTraceUrls.forEach(url => addDependencyOnUrl(node, url)); - break; - case 'TimerFire': { - // @ts-expect-error - 'TimerFire' event means timerId exists. - const installer = timers.get(evt.args.data.timerId); - if (!installer || installer.endTime > node.startTime) break; - installer.addDependent(node); - break; - } - - case 'InvalidateLayout': - case 'ScheduleStyleRecalculation': - addDependencyOnFrame(node, evt.args.data.frame); - stackTraceUrls.forEach(url => addDependencyOnUrl(node, url)); - break; - - case 'EvaluateScript': - addDependencyOnFrame(node, evt.args.data.frame); - // @ts-expect-error - 'EvaluateScript' event means argsUrl is defined. - addDependencyOnUrl(node, argsUrl); - stackTraceUrls.forEach(url => addDependencyOnUrl(node, url)); - break; - - case 'XHRReadyStateChange': - // Only create the dependency if the request was completed - // 'XHRReadyStateChange' event means readyState is defined. - if (evt.args.data.readyState !== 4) break; - - // @ts-expect-error - 'XHRReadyStateChange' event means argsUrl is defined. - addDependencyOnUrl(node, argsUrl); - stackTraceUrls.forEach(url => addDependencyOnUrl(node, url)); - break; - - case 'FunctionCall': - case 'v8.compile': - addDependencyOnFrame(node, evt.args.data.frame); - // @ts-expect-error - events mean argsUrl is defined. - addDependencyOnUrl(node, argsUrl); - break; - - case 'ParseAuthorStyleSheet': - addDependencyOnFrame(node, evt.args.data.frame); - // @ts-expect-error - 'ParseAuthorStyleSheet' event means styleSheetUrl is defined. - addDependencyOnUrl(node, evt.args.data.styleSheetUrl); - break; - - case 'ResourceSendRequest': - addDependencyOnFrame(node, evt.args.data.frame); - // @ts-expect-error - 'ResourceSendRequest' event means requestId is defined. - addDependentNetworkRequest(node, evt.args.data.requestId); - stackTraceUrls.forEach(url => addDependencyOnUrl(node, url)); - break; - } - } - - // Nodes starting before the root node cannot depend on it. - if (node.getNumberOfDependencies() === 0 && node.canDependOn(rootNode)) { - node.addDependency(rootNode); - } - } - - // Second pass to prune the graph of short tasks. - const minimumEvtDur = SIGNIFICANT_DUR_THRESHOLD_MS * 1000; - let foundFirstLayout = false; - let foundFirstPaint = false; - let foundFirstParse = false; - - for (const node of cpuNodes) { - // Don't prune if event is the first ParseHTML/Layout/Paint. - // See https://github.com/GoogleChrome/lighthouse/issues/9627#issuecomment-526699524 for more. - let isFirst = false; - if (!foundFirstLayout && node.childEvents.some(evt => evt.name === 'Layout')) { - isFirst = foundFirstLayout = true; - } - if (!foundFirstPaint && node.childEvents.some(evt => evt.name === 'Paint')) { - isFirst = foundFirstPaint = true; - } - if (!foundFirstParse && node.childEvents.some(evt => evt.name === 'ParseHTML')) { - isFirst = foundFirstParse = true; - } - - if (isFirst || node.event.dur >= minimumEvtDur) { - // Don't prune this node. The task is long / important so it will impact simulation. - continue; - } - - // Prune the node if it isn't highly connected to minimize graph size. Rewiring the graph - // here replaces O(M + N) edges with (M * N) edges, which is fine if either M or N is at - // most 1. - if (node.getNumberOfDependencies() === 1 || node.getNumberOfDependents() <= 1) { - PageDependencyGraph._pruneNode(node); - } - } - } - - /** - * Removes the given node from the graph, but retains all paths between its dependencies and - * dependents. - * @param {Node} node - */ - static _pruneNode(node) { - const dependencies = node.getDependencies(); - const dependents = node.getDependents(); - for (const dependency of dependencies) { - node.removeDependency(dependency); - for (const dependent of dependents) { - dependency.addDependent(dependent); - } - } - for (const dependent of dependents) { - node.removeDependent(dependent); - } - } - - /** - * @param {LH.Artifacts.ProcessedTrace} processedTrace - * @param {Array} networkRecords - * @param {URLArtifact} URL - * @return {Node} - */ - static createGraph(processedTrace, networkRecords, URL) { - const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords); - const cpuNodes = PageDependencyGraph.getCPUNodes(processedTrace); - const {requestedUrl, mainDocumentUrl} = URL; - if (!requestedUrl) throw new Error('requestedUrl is required to get the root request'); - if (!mainDocumentUrl) throw new Error('mainDocumentUrl is required to get the main resource'); - - const rootRequest = NetworkAnalyzer.findResourceForUrl(networkRecords, requestedUrl); - if (!rootRequest) throw new Error('rootRequest not found'); - const rootNode = networkNodeOutput.idToNodeMap.get(rootRequest.requestId); - if (!rootNode) throw new Error('rootNode not found'); - - const mainDocumentRequest = - NetworkAnalyzer.findLastDocumentForUrl(networkRecords, mainDocumentUrl); - if (!mainDocumentRequest) throw new Error('mainDocumentRequest not found'); - const mainDocumentNode = networkNodeOutput.idToNodeMap.get(mainDocumentRequest.requestId); - if (!mainDocumentNode) throw new Error('mainDocumentNode not found'); - - PageDependencyGraph.linkNetworkNodes(rootNode, networkNodeOutput); - PageDependencyGraph.linkCPUNodes(rootNode, networkNodeOutput, cpuNodes); - mainDocumentNode.setIsMainDocument(true); - - if (NetworkNode.hasCycle(rootNode)) { - throw new Error('Invalid dependency graph created, cycle detected'); - } - - return rootNode; - } - - /** - * - * @param {Node} rootNode - */ - static printGraph(rootNode, widthInCharacters = 100) { - /** @param {string} str @param {number} target */ - function padRight(str, target, padChar = ' ') { - return str + padChar.repeat(Math.max(target - str.length, 0)); - } - - /** @type {Array} */ - const nodes = []; - rootNode.traverse(node => nodes.push(node)); - nodes.sort((a, b) => a.startTime - b.startTime); - - const min = nodes[0].startTime; - const max = nodes.reduce((max, node) => Math.max(max, node.endTime), 0); - - const totalTime = max - min; - const timePerCharacter = totalTime / widthInCharacters; - nodes.forEach(node => { - const offset = Math.round((node.startTime - min) / timePerCharacter); - const length = Math.ceil((node.endTime - node.startTime) / timePerCharacter); - const bar = padRight('', offset) + padRight('', length, '='); - - // @ts-expect-error -- disambiguate displayName from across possible Node types. - const displayName = node.record ? node.record.url : node.type; - // eslint-disable-next-line - console.log(padRight(bar, widthInCharacters), `| ${displayName.slice(0, 30)}`); - }); - } - /** * @param {{trace: LH.Trace, devtoolsLog: LH.DevtoolsLog, URL: LH.Artifacts['URL']}} data * @param {LH.Artifacts.ComputedContext} context @@ -478,7 +30,8 @@ class PageDependencyGraph { // Calculates the URL artifact from the processed trace and DT log. const URL = data.URL || await DocumentUrls.request(data, context); - return PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const lanternRequests = networkRecords.map(NetworkRequest.asLanternNetworkRequest); + return LanternPageDependencyGraph.createGraph(processedTrace, lanternRequests, URL); } } diff --git a/core/lib/lantern/page-dependency-graph.js b/core/lib/lantern/page-dependency-graph.js new file mode 100644 index 000000000000..3a1de4d8a94e --- /dev/null +++ b/core/lib/lantern/page-dependency-graph.js @@ -0,0 +1,463 @@ +/** + * @license + * Copyright 2024 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import {NetworkRequestTypes} from './lantern.js'; +import {NetworkNode} from './network-node.js'; +import {CPUNode} from './cpu-node.js'; +import {TraceProcessor} from '../tracehouse/trace-processor.js'; +import {NetworkAnalyzer} from './simulator/network-analyzer.js'; + +/** @typedef {import('../../../types/internal/lantern.js').Lantern.NetworkRequest} NetworkRequest */ +/** @typedef {import('./base-node.js').Node} Node */ +/** @typedef {Omit} URLArtifact */ + +/** + * @typedef {Object} NetworkNodeOutput + * @property {Array} nodes + * @property {Map} idToNodeMap + * @property {Map>} urlToNodeMap + * @property {Map} frameIdToNodeMap + */ + +// Shorter tasks have negligible impact on simulation results. +const SIGNIFICANT_DUR_THRESHOLD_MS = 10; + +// TODO: video files tend to be enormous and throw off all graph traversals, move this ignore +// into estimation logic when we use the dependency graph for other purposes. +const IGNORED_MIME_TYPES_REGEX = /^video/; + +class PageDependencyGraph { + /** + * @param {NetworkRequest} record + * @return {Array} + */ + static getNetworkInitiators(record) { + if (!record.initiator) return []; + if (record.initiator.url) return [record.initiator.url]; + if (record.initiator.type === 'script') { + // Script initiators have the stack of callFrames from all functions that led to this request. + // If async stacks are enabled, then the stack will also have the parent functions that asynchronously + // led to this request chained in the `parent` property. + /** @type {Set} */ + const scriptURLs = new Set(); + let stack = record.initiator.stack; + while (stack) { + const callFrames = stack.callFrames || []; + for (const frame of callFrames) { + if (frame.url) scriptURLs.add(frame.url); + } + + stack = stack.parent; + } + + return Array.from(scriptURLs); + } + + return []; + } + + /** + * @param {Array} networkRecords + * @return {NetworkNodeOutput} + */ + static getNetworkNodeOutput(networkRecords) { + /** @type {Array} */ + const nodes = []; + /** @type {Map} */ + const idToNodeMap = new Map(); + /** @type {Map>} */ + const urlToNodeMap = new Map(); + /** @type {Map} */ + const frameIdToNodeMap = new Map(); + + networkRecords.forEach(record => { + if (IGNORED_MIME_TYPES_REGEX.test(record.mimeType)) return; + if (record.sessionTargetType === 'worker') return; + + // Network record requestIds can be duplicated for an unknown reason + // Suffix all subsequent records with `:duplicate` until it's unique + // NOTE: This should never happen with modern NetworkRequest library, but old fixtures + // might still have this issue. + while (idToNodeMap.has(record.requestId)) { + record.requestId += ':duplicate'; + } + + const node = new NetworkNode(record); + nodes.push(node); + + const urlList = urlToNodeMap.get(record.url) || []; + urlList.push(node); + + idToNodeMap.set(record.requestId, node); + urlToNodeMap.set(record.url, urlList); + + // If the request was for the root document of an iframe, save an entry in our + // map so we can link up the task `args.data.frame` dependencies later in graph creation. + if (record.frameId && + record.resourceType === NetworkRequestTypes.Document && + record.documentURL === record.url) { + // If there's ever any ambiguity, permanently set the value to `false` to avoid loops in the graph. + const value = frameIdToNodeMap.has(record.frameId) ? null : node; + frameIdToNodeMap.set(record.frameId, value); + } + }); + + return {nodes, idToNodeMap, urlToNodeMap, frameIdToNodeMap}; + } + + /** + * @param {LH.Artifacts.ProcessedTrace} processedTrace + * @return {Array} + */ + static getCPUNodes({mainThreadEvents}) { + /** @type {Array} */ + const nodes = []; + let i = 0; + + TraceProcessor.assertHasToplevelEvents(mainThreadEvents); + + while (i < mainThreadEvents.length) { + const evt = mainThreadEvents[i]; + i++; + + // Skip all trace events that aren't schedulable tasks with sizable duration + if (!TraceProcessor.isScheduleableTask(evt) || !evt.dur) { + continue; + } + + // Capture all events that occurred within the task + /** @type {Array} */ + const children = []; + for ( + const endTime = evt.ts + evt.dur; + i < mainThreadEvents.length && mainThreadEvents[i].ts < endTime; + i++ + ) { + children.push(mainThreadEvents[i]); + } + + nodes.push(new CPUNode(evt, children)); + } + + return nodes; + } + + /** + * @param {NetworkNode} rootNode + * @param {NetworkNodeOutput} networkNodeOutput + */ + static linkNetworkNodes(rootNode, networkNodeOutput) { + networkNodeOutput.nodes.forEach(node => { + const directInitiatorRequest = node.request.initiatorRequest || rootNode.request; + const directInitiatorNode = + networkNodeOutput.idToNodeMap.get(directInitiatorRequest.requestId) || rootNode; + const canDependOnInitiator = + !directInitiatorNode.isDependentOn(node) && + node.canDependOn(directInitiatorNode); + const initiators = PageDependencyGraph.getNetworkInitiators(node.request); + if (initiators.length) { + initiators.forEach(initiator => { + const parentCandidates = networkNodeOutput.urlToNodeMap.get(initiator) || []; + // Only add the edge if the parent is unambiguous with valid timing and isn't circular. + if (parentCandidates.length === 1 && + parentCandidates[0].startTime <= node.startTime && + !parentCandidates[0].isDependentOn(node)) { + node.addDependency(parentCandidates[0]); + } else if (canDependOnInitiator) { + directInitiatorNode.addDependent(node); + } + }); + } else if (canDependOnInitiator) { + directInitiatorNode.addDependent(node); + } + + // Make sure the nodes are attached to the graph if the initiator information was invalid. + if (node !== rootNode && node.getDependencies().length === 0 && node.canDependOn(rootNode)) { + node.addDependency(rootNode); + } + + if (!node.request.redirects) return; + + const redirects = [...node.request.redirects, node.request]; + for (let i = 1; i < redirects.length; i++) { + const redirectNode = networkNodeOutput.idToNodeMap.get(redirects[i - 1].requestId); + const actualNode = networkNodeOutput.idToNodeMap.get(redirects[i].requestId); + if (actualNode && redirectNode) { + actualNode.addDependency(redirectNode); + } + } + }); + } + + /** + * @param {Node} rootNode + * @param {NetworkNodeOutput} networkNodeOutput + * @param {Array} cpuNodes + */ + static linkCPUNodes(rootNode, networkNodeOutput, cpuNodes) { + /** @type {Set} */ + const linkableResourceTypes = new Set([ + NetworkRequestTypes.XHR, NetworkRequestTypes.Fetch, NetworkRequestTypes.Script, + ]); + + /** @param {CPUNode} cpuNode @param {string} reqId */ + function addDependentNetworkRequest(cpuNode, reqId) { + const networkNode = networkNodeOutput.idToNodeMap.get(reqId); + if (!networkNode || + // Ignore all network nodes that started before this CPU task started + // A network request that started earlier could not possibly have been started by this task + networkNode.startTime <= cpuNode.startTime) return; + const {request} = networkNode; + const resourceType = request.resourceType || + request.redirectDestination?.resourceType; + if (!linkableResourceTypes.has(resourceType)) { + // We only link some resources to CPU nodes because we observe LCP simulation + // regressions when including images, etc. + return; + } + cpuNode.addDependent(networkNode); + } + + /** + * If the node has an associated frameId, then create a dependency on the root document request + * for the frame. The task obviously couldn't have started before the frame was even downloaded. + * + * @param {CPUNode} cpuNode + * @param {string|undefined} frameId + */ + function addDependencyOnFrame(cpuNode, frameId) { + if (!frameId) return; + const networkNode = networkNodeOutput.frameIdToNodeMap.get(frameId); + if (!networkNode) return; + // Ignore all network nodes that started after this CPU task started + // A network request that started after could not possibly be required this task + if (networkNode.startTime >= cpuNode.startTime) return; + cpuNode.addDependency(networkNode); + } + + /** @param {CPUNode} cpuNode @param {string} url */ + function addDependencyOnUrl(cpuNode, url) { + if (!url) return; + // Allow network requests that end up to 100ms before the task started + // Some script evaluations can start before the script finishes downloading + const minimumAllowableTimeSinceNetworkNodeEnd = -100 * 1000; + const candidates = networkNodeOutput.urlToNodeMap.get(url) || []; + + let minCandidate = null; + let minDistance = Infinity; + // Find the closest request that finished before this CPU task started + for (const candidate of candidates) { + // Explicitly ignore all requests that started after this CPU node + // A network request that started after this task started cannot possibly be a dependency + if (cpuNode.startTime <= candidate.startTime) return; + + const distance = cpuNode.startTime - candidate.endTime; + if (distance >= minimumAllowableTimeSinceNetworkNodeEnd && distance < minDistance) { + minCandidate = candidate; + minDistance = distance; + } + } + + if (!minCandidate) return; + cpuNode.addDependency(minCandidate); + } + + /** @type {Map} */ + const timers = new Map(); + for (const node of cpuNodes) { + for (const evt of node.childEvents) { + if (!evt.args.data) continue; + + const argsUrl = evt.args.data.url; + const stackTraceUrls = (evt.args.data.stackTrace || []).map(l => l.url).filter(Boolean); + + switch (evt.name) { + case 'TimerInstall': + // @ts-expect-error - 'TimerInstall' event means timerId exists. + timers.set(evt.args.data.timerId, node); + stackTraceUrls.forEach(url => addDependencyOnUrl(node, url)); + break; + case 'TimerFire': { + // @ts-expect-error - 'TimerFire' event means timerId exists. + const installer = timers.get(evt.args.data.timerId); + if (!installer || installer.endTime > node.startTime) break; + installer.addDependent(node); + break; + } + + case 'InvalidateLayout': + case 'ScheduleStyleRecalculation': + addDependencyOnFrame(node, evt.args.data.frame); + stackTraceUrls.forEach(url => addDependencyOnUrl(node, url)); + break; + + case 'EvaluateScript': + addDependencyOnFrame(node, evt.args.data.frame); + // @ts-expect-error - 'EvaluateScript' event means argsUrl is defined. + addDependencyOnUrl(node, argsUrl); + stackTraceUrls.forEach(url => addDependencyOnUrl(node, url)); + break; + + case 'XHRReadyStateChange': + // Only create the dependency if the request was completed + // 'XHRReadyStateChange' event means readyState is defined. + if (evt.args.data.readyState !== 4) break; + + // @ts-expect-error - 'XHRReadyStateChange' event means argsUrl is defined. + addDependencyOnUrl(node, argsUrl); + stackTraceUrls.forEach(url => addDependencyOnUrl(node, url)); + break; + + case 'FunctionCall': + case 'v8.compile': + addDependencyOnFrame(node, evt.args.data.frame); + // @ts-expect-error - events mean argsUrl is defined. + addDependencyOnUrl(node, argsUrl); + break; + + case 'ParseAuthorStyleSheet': + addDependencyOnFrame(node, evt.args.data.frame); + // @ts-expect-error - 'ParseAuthorStyleSheet' event means styleSheetUrl is defined. + addDependencyOnUrl(node, evt.args.data.styleSheetUrl); + break; + + case 'ResourceSendRequest': + addDependencyOnFrame(node, evt.args.data.frame); + // @ts-expect-error - 'ResourceSendRequest' event means requestId is defined. + addDependentNetworkRequest(node, evt.args.data.requestId); + stackTraceUrls.forEach(url => addDependencyOnUrl(node, url)); + break; + } + } + + // Nodes starting before the root node cannot depend on it. + if (node.getNumberOfDependencies() === 0 && node.canDependOn(rootNode)) { + node.addDependency(rootNode); + } + } + + // Second pass to prune the graph of short tasks. + const minimumEvtDur = SIGNIFICANT_DUR_THRESHOLD_MS * 1000; + let foundFirstLayout = false; + let foundFirstPaint = false; + let foundFirstParse = false; + + for (const node of cpuNodes) { + // Don't prune if event is the first ParseHTML/Layout/Paint. + // See https://github.com/GoogleChrome/lighthouse/issues/9627#issuecomment-526699524 for more. + let isFirst = false; + if (!foundFirstLayout && node.childEvents.some(evt => evt.name === 'Layout')) { + isFirst = foundFirstLayout = true; + } + if (!foundFirstPaint && node.childEvents.some(evt => evt.name === 'Paint')) { + isFirst = foundFirstPaint = true; + } + if (!foundFirstParse && node.childEvents.some(evt => evt.name === 'ParseHTML')) { + isFirst = foundFirstParse = true; + } + + if (isFirst || node.event.dur >= minimumEvtDur) { + // Don't prune this node. The task is long / important so it will impact simulation. + continue; + } + + // Prune the node if it isn't highly connected to minimize graph size. Rewiring the graph + // here replaces O(M + N) edges with (M * N) edges, which is fine if either M or N is at + // most 1. + if (node.getNumberOfDependencies() === 1 || node.getNumberOfDependents() <= 1) { + PageDependencyGraph._pruneNode(node); + } + } + } + + /** + * Removes the given node from the graph, but retains all paths between its dependencies and + * dependents. + * @param {Node} node + */ + static _pruneNode(node) { + const dependencies = node.getDependencies(); + const dependents = node.getDependents(); + for (const dependency of dependencies) { + node.removeDependency(dependency); + for (const dependent of dependents) { + dependency.addDependent(dependent); + } + } + for (const dependent of dependents) { + node.removeDependent(dependent); + } + } + + /** + * @param {LH.Artifacts.ProcessedTrace} processedTrace + * @param {Array} networkRecords + * @param {URLArtifact} URL + * @return {Node} + */ + static createGraph(processedTrace, networkRecords, URL) { + const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords); + const cpuNodes = PageDependencyGraph.getCPUNodes(processedTrace); + const {requestedUrl, mainDocumentUrl} = URL; + if (!requestedUrl) throw new Error('requestedUrl is required to get the root request'); + if (!mainDocumentUrl) throw new Error('mainDocumentUrl is required to get the main resource'); + + const rootRequest = NetworkAnalyzer.findResourceForUrl(networkRecords, requestedUrl); + if (!rootRequest) throw new Error('rootRequest not found'); + const rootNode = networkNodeOutput.idToNodeMap.get(rootRequest.requestId); + if (!rootNode) throw new Error('rootNode not found'); + + const mainDocumentRequest = + NetworkAnalyzer.findLastDocumentForUrl(networkRecords, mainDocumentUrl); + if (!mainDocumentRequest) throw new Error('mainDocumentRequest not found'); + const mainDocumentNode = networkNodeOutput.idToNodeMap.get(mainDocumentRequest.requestId); + if (!mainDocumentNode) throw new Error('mainDocumentNode not found'); + + PageDependencyGraph.linkNetworkNodes(rootNode, networkNodeOutput); + PageDependencyGraph.linkCPUNodes(rootNode, networkNodeOutput, cpuNodes); + mainDocumentNode.setIsMainDocument(true); + + if (NetworkNode.hasCycle(rootNode)) { + throw new Error('Invalid dependency graph created, cycle detected'); + } + + return rootNode; + } + + /** + * + * @param {Node} rootNode + */ + static printGraph(rootNode, widthInCharacters = 100) { + /** @param {string} str @param {number} target */ + function padRight(str, target, padChar = ' ') { + return str + padChar.repeat(Math.max(target - str.length, 0)); + } + + /** @type {Array} */ + const nodes = []; + rootNode.traverse(node => nodes.push(node)); + nodes.sort((a, b) => a.startTime - b.startTime); + + const min = nodes[0].startTime; + const max = nodes.reduce((max, node) => Math.max(max, node.endTime), 0); + + const totalTime = max - min; + const timePerCharacter = totalTime / widthInCharacters; + nodes.forEach(node => { + const offset = Math.round((node.startTime - min) / timePerCharacter); + const length = Math.ceil((node.endTime - node.startTime) / timePerCharacter); + const bar = padRight('', offset) + padRight('', length, '='); + + // @ts-expect-error -- disambiguate displayName from across possible Node types. + const displayName = node.request ? node.request.url : node.type; + // eslint-disable-next-line + console.log(padRight(bar, widthInCharacters), `| ${displayName.slice(0, 30)}`); + }); + } +} + +export {PageDependencyGraph}; diff --git a/core/lib/network-recorder.js b/core/lib/network-recorder.js index fa0b3af8bd4c..ee424fdc1e64 100644 --- a/core/lib/network-recorder.js +++ b/core/lib/network-recorder.js @@ -10,7 +10,7 @@ import log from 'lighthouse-logger'; import * as LH from '../../types/lh.js'; import {NetworkRequest} from './network-request.js'; -import {PageDependencyGraph} from '../computed/page-dependency-graph.js'; +import {PageDependencyGraph} from '../lib/lantern/page-dependency-graph.js'; /** * @typedef {{ diff --git a/core/test/computed/page-dependency-graph-test.js b/core/test/computed/page-dependency-graph-test.js index 0f7cece23e2f..180eaaa26d1f 100644 --- a/core/test/computed/page-dependency-graph-test.js +++ b/core/test/computed/page-dependency-graph-test.js @@ -8,64 +8,12 @@ import assert from 'assert/strict'; import {PageDependencyGraph} from '../../computed/page-dependency-graph.js'; import {BaseNode} from '../../lib/lantern/base-node.js'; -import {NetworkRequest} from '../../lib/network-request.js'; import {getURLArtifactFromDevtoolsLog, readJson} from '../test-utils.js'; const sampleTrace = readJson('../fixtures/traces/iframe-m79.trace.json', import.meta); const sampleDevtoolsLog = readJson('../fixtures/traces/iframe-m79.devtoolslog.json', import.meta); -function createRequest( - requestId, - url, - rendererStartTime = 0, - initiator = null, - resourceType = NetworkRequest.TYPES.Document, - sessionTargetType = 'page' -) { - const networkEndTime = rendererStartTime + 50; - return { - requestId, - url, - rendererStartTime, - networkEndTime, - initiator, - resourceType, - sessionTargetType, - }; -} - -const TOPLEVEL_TASK_NAME = 'TaskQueueManager::ProcessTaskFromWorkQueue'; -describe('PageDependencyGraph computed artifact:', () => { - let processedTrace; - let URL; - - function addTaskEvents(startTs, duration, evts) { - const mainEvent = { - name: TOPLEVEL_TASK_NAME, - tid: 1, - ts: startTs * 1000, - dur: duration * 1000, - args: {}, - }; - - processedTrace.mainThreadEvents.push(mainEvent); - - let i = 0; - for (const evt of evts) { - i++; - processedTrace.mainThreadEvents.push({ - name: evt.name, - ts: (evt.ts * 1000) || (startTs * 1000 + i), - args: {data: evt.data}, - }); - } - } - - beforeEach(() => { - processedTrace = {mainThreadEvents: []}; - URL = {requestedUrl: 'https://example.com/', mainDocumentUrl: 'https://example.com/'}; - }); - +describe('PageDependencyGraph computed artifact', () => { describe('#compute_', () => { it('should compute the dependency graph', () => { const context = {computedCache: new Map()}; @@ -96,564 +44,4 @@ describe('PageDependencyGraph computed artifact:', () => { }); }); }); - - describe('#getNetworkNodeOutput', () => { - const request1 = createRequest(1, 'https://example.com/'); - const request2 = createRequest(2, 'https://example.com/page'); - const request3 = createRequest(3, 'https://example.com/page'); - const networkRecords = [request1, request2, request3]; - - it('should create network nodes', () => { - const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords); - for (let i = 0; i < networkRecords.length; i++) { - const node = networkNodeOutput.nodes[i]; - assert.ok(node, `did not create node at index ${i}`); - assert.equal(node.id, i + 1); - assert.equal(node.type, 'network'); - assert.equal(node.record, networkRecords[i]); - } - }); - - it('should ignore worker requests', () => { - const workerRequest = createRequest(4, 'https://example.com/worker.js', 0, null, 'Script', 'worker'); - const recordsWithWorker = [ - ...networkRecords, - workerRequest, - ]; - - const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(recordsWithWorker); - - expect(networkNodeOutput.nodes).toHaveLength(3); - expect(networkNodeOutput.nodes.map(node => node.record)).not.toContain(workerRequest); - }); - - it('should index nodes by ID', () => { - const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords); - const indexedById = networkNodeOutput.idToNodeMap; - for (const record of networkRecords) { - assert.equal(indexedById.get(record.requestId).record, record); - } - }); - - it('should index nodes by URL', () => { - const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords); - const nodes = networkNodeOutput.nodes; - const indexedByUrl = networkNodeOutput.urlToNodeMap; - assert.deepEqual(indexedByUrl.get('https://example.com/'), [nodes[0]]); - assert.deepEqual(indexedByUrl.get('https://example.com/page'), [nodes[1], nodes[2]]); - }); - - it('should index nodes by frame', () => { - const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput([ - {...createRequest(1, 'https://example.com/'), documentURL: 'https://example.com/', frameId: 'A'}, - {...createRequest(2, 'https://example.com/page'), documentURL: 'https://example.com/', frameId: 'A'}, - {...createRequest(3, 'https://example.com/page2'), documentURL: 'https://example.com/page2', frameId: 'C', - resourceType: NetworkRequest.TYPES.XHR}, - {...createRequest(4, 'https://example.com/page3'), documentURL: 'https://example.com/page3', frameId: 'D'}, - {...createRequest(4, 'https://example.com/page4'), documentURL: 'https://example.com/page4', frameId: undefined}, - {...createRequest(4, 'https://example.com/page5'), documentURL: 'https://example.com/page5', frameId: 'collision'}, - {...createRequest(4, 'https://example.com/page6'), documentURL: 'https://example.com/page6', frameId: 'collision'}, - ]); - - const nodes = networkNodeOutput.nodes; - const indexedByFrame = networkNodeOutput.frameIdToNodeMap; - expect([...indexedByFrame.entries()]).toEqual([ - ['A', nodes[0]], - ['D', nodes[3]], - ['collision', null], - ]); - }); - }); - - describe('#getCPUNodes', () => { - it('should create CPU nodes', () => { - addTaskEvents(0, 100, [ - {name: 'MyCustomEvent'}, - {name: 'OtherEvent'}, - {name: 'OutsideTheWindow', ts: 200}, - {name: 'OrphanedEvent'}, // should be ignored since we stopped at OutsideTheWindow - ]); - - addTaskEvents(250, 50, [ - {name: 'LaterEvent'}, - ]); - - assert.equal(processedTrace.mainThreadEvents.length, 7); - const nodes = PageDependencyGraph.getCPUNodes(processedTrace); - assert.equal(nodes.length, 2); - - const node1 = nodes[0]; - assert.equal(node1.id, '1.0'); - assert.equal(node1.type, 'cpu'); - assert.equal(node1.event, processedTrace.mainThreadEvents[0]); - assert.equal(node1.childEvents.length, 2); - assert.equal(node1.childEvents[1].name, 'OtherEvent'); - - const node2 = nodes[1]; - assert.equal(node2.id, '1.250000'); - assert.equal(node2.type, 'cpu'); - assert.equal(node2.event, processedTrace.mainThreadEvents[5]); - assert.equal(node2.childEvents.length, 1); - assert.equal(node2.childEvents[0].name, 'LaterEvent'); - }); - }); - - describe('#createGraph', () => { - it('should compute a simple network graph', () => { - const request1 = createRequest(1, 'https://example.com/', 0); - const request2 = createRequest(2, 'https://example.com/page', 5); - const request3 = createRequest(3, 'https://example.com/page2', 5); - const request4 = createRequest(4, 'https://example.com/page3', 10, {url: 'https://example.com/page'}); - const networkRecords = [request1, request2, request3, request4]; - - addTaskEvents(0, 0, []); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - assert.equal(nodes.length, 4); - assert.deepEqual(nodes.map(node => node.id), [1, 2, 3, 4]); - assert.deepEqual(nodes[0].getDependencies(), []); - assert.deepEqual(nodes[1].getDependencies(), [nodes[0]]); - assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); - assert.deepEqual(nodes[3].getDependencies(), [nodes[1]]); - }); - - it('should compute a simple network and CPU graph', () => { - const request1 = createRequest(1, 'https://example.com/', 0); - const request2 = createRequest(2, 'https://example.com/page', 50); - const request3 = createRequest(3, 'https://example.com/page2', 50); - const request4 = createRequest(4, 'https://example.com/page3', 300, null, NetworkRequest.TYPES.XHR); - const networkRecords = [request1, request2, request3, request4]; - - addTaskEvents(200, 200, [ - {name: 'EvaluateScript', data: {url: 'https://example.com/page'}}, - {name: 'ResourceSendRequest', data: {requestId: 4}}, - ]); - - addTaskEvents(700, 50, [ - {name: 'InvalidateLayout', data: {stackTrace: [{url: 'https://example.com/page2'}]}}, - {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page3'}}, - ]); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - const getIds = nodes => nodes.map(node => node.id); - const getDependencyIds = node => getIds(node.getDependencies()); - - assert.equal(nodes.length, 6); - assert.deepEqual(getIds(nodes), [1, 2, 3, 4, '1.200000', '1.700000']); - assert.deepEqual(getDependencyIds(nodes[0]), []); - assert.deepEqual(getDependencyIds(nodes[1]), [1]); - assert.deepEqual(getDependencyIds(nodes[2]), [1]); - assert.deepEqual(getDependencyIds(nodes[3]), [1, '1.200000']); - assert.deepEqual(getDependencyIds(nodes[4]), [2]); - assert.deepEqual(getDependencyIds(nodes[5]), [3, 4]); - }); - - it('should compute a network graph with duplicate URLs', () => { - const request1 = createRequest(1, 'https://example.com/', 0); - const request2 = createRequest(2, 'https://example.com/page', 5); - const request3 = createRequest(3, 'https://example.com/page', 5); // duplicate URL - const request4 = createRequest(4, 'https://example.com/page3', 10, {url: 'https://example.com/page'}); - const networkRecords = [request1, request2, request3, request4]; - - addTaskEvents(0, 0, []); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - assert.equal(nodes.length, 4); - assert.deepEqual(nodes.map(node => node.id), [1, 2, 3, 4]); - assert.deepEqual(nodes[0].getDependencies(), []); - assert.deepEqual(nodes[1].getDependencies(), [nodes[0]]); - assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); - assert.deepEqual(nodes[3].getDependencies(), [nodes[0]]); // should depend on rootNode instead - }); - - it('should be forgiving without cyclic dependencies', () => { - const request1 = createRequest(1, 'https://example.com/', 0); - const request2 = createRequest(2, 'https://example.com/page', 250, null, NetworkRequest.TYPES.XHR); - const request3 = createRequest(3, 'https://example.com/page2', 210); - const request4 = createRequest(4, 'https://example.com/page3', 590); - const request5 = createRequest(5, 'https://example.com/page4', 595, null, NetworkRequest.TYPES.XHR); - const networkRecords = [request1, request2, request3, request4, request5]; - - addTaskEvents(200, 200, [ - // CPU 1.2 should depend on Network 1 - {name: 'EvaluateScript', data: {url: 'https://example.com/'}}, - - // Network 2 should depend on CPU 1.2, but 1.2 should not depend on Network 1 - {name: 'ResourceSendRequest', data: {requestId: 2}}, - {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page'}}, - - // CPU 1.2 should not depend on Network 3 because it starts after CPU 1.2 - {name: 'EvaluateScript', data: {url: 'https://example.com/page2'}}, - ]); - - addTaskEvents(600, 150, [ - // CPU 1.6 should depend on Network 4 even though it ends at 410ms - {name: 'InvalidateLayout', data: {stackTrace: [{url: 'https://example.com/page3'}]}}, - // Network 5 should not depend on CPU 1.6 because it started before CPU 1.6 - {name: 'ResourceSendRequest', data: {requestId: 5}}, - ]); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - const getDependencyIds = node => node.getDependencies().map(node => node.id); - - assert.equal(nodes.length, 7); - assert.deepEqual(getDependencyIds(nodes[0]), []); - assert.deepEqual(getDependencyIds(nodes[1]), [1, '1.200000']); - assert.deepEqual(getDependencyIds(nodes[2]), [1]); - assert.deepEqual(getDependencyIds(nodes[3]), [1]); - assert.deepEqual(getDependencyIds(nodes[4]), [1]); - assert.deepEqual(getDependencyIds(nodes[5]), [1]); - assert.deepEqual(getDependencyIds(nodes[6]), [4]); - }); - - it('should not install timer dependency on itself', () => { - const request1 = createRequest(1, 'https://example.com/', 0); - const networkRecords = [request1]; - - addTaskEvents(200, 200, [ - // CPU 1.2 should depend on Network 1 - {name: 'EvaluateScript', data: {url: 'https://example.com/'}}, - // CPU 1.2 will install and fire it's own timer, but should not depend on itself - {name: 'TimerInstall', data: {timerId: 'timer1'}}, - {name: 'TimerFire', data: {timerId: 'timer1'}}, - ]); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - const getDependencyIds = node => node.getDependencies().map(node => node.id); - - assert.equal(nodes.length, 2); - assert.deepEqual(getDependencyIds(nodes[0]), []); - assert.deepEqual(getDependencyIds(nodes[1]), [1]); - }); - - it('should prune short tasks', () => { - const request0 = createRequest(0, 'https://example.com/page0', 0); - const request1 = createRequest(1, 'https://example.com/', 100, null, NetworkRequest.TYPES.Script); - const request2 = createRequest(2, 'https://example.com/page', 200, null, NetworkRequest.TYPES.XHR); - const request3 = createRequest(3, 'https://example.com/page2', 300, null, NetworkRequest.TYPES.Script); - const request4 = createRequest(4, 'https://example.com/page3', 400, null, NetworkRequest.TYPES.XHR); - const networkRecords = [request0, request1, request2, request3, request4]; - URL = {requestedUrl: 'https://example.com/page0', mainDocumentUrl: 'https://example.com/page0'}; - - // Long task, should be kept in the output. - addTaskEvents(120, 50, [ - {name: 'EvaluateScript', data: {url: 'https://example.com/'}}, - {name: 'ResourceSendRequest', data: {requestId: 2}}, - {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page'}}, - ]); - - // Short task, should be pruned, but the 3->4 relationship should be retained - addTaskEvents(350, 5, [ - {name: 'EvaluateScript', data: {url: 'https://example.com/page2'}}, - {name: 'ResourceSendRequest', data: {requestId: 4}}, - {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page3'}}, - ]); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - const getDependencyIds = node => node.getDependencies().map(node => node.id); - - assert.equal(nodes.length, 6); - - assert.deepEqual(getDependencyIds(nodes[0]), []); - assert.deepEqual(getDependencyIds(nodes[1]), [0]); - assert.deepEqual(getDependencyIds(nodes[2]), [0, '1.120000']); - assert.deepEqual(getDependencyIds(nodes[3]), [0]); - assert.deepEqual(getDependencyIds(nodes[4]), [0, 3]); - - assert.equal('1.120000', nodes[5].id); - assert.deepEqual(getDependencyIds(nodes[5]), [1]); - }); - - it('should not prune highly-connected short tasks', () => { - const request0 = createRequest(0, 'https://example.com/page0', 0); - const request1 = { - ...createRequest(1, 'https://example.com/', 100, null, NetworkRequest.TYPES.Document), - documentURL: 'https://example.com/', - frameId: 'frame1', - }; - const request2 = { - ...createRequest(2, 'https://example.com/page', 200, null, NetworkRequest.TYPES.Script), - documentURL: 'https://example.com/', - frameId: 'frame1', - }; - const request3 = createRequest(3, 'https://example.com/page2', 300, null, NetworkRequest.TYPES.XHR); - const request4 = createRequest(4, 'https://example.com/page3', 400, null, NetworkRequest.TYPES.XHR); - const networkRecords = [request0, request1, request2, request3, request4]; - URL = {requestedUrl: 'https://example.com/page0', mainDocumentUrl: 'https://example.com/page0'}; - - // Short task, evaluates script (2) and sends two XHRs. - addTaskEvents(220, 5, [ - {name: 'EvaluateScript', data: {url: 'https://example.com/page', frame: 'frame1'}}, - - {name: 'ResourceSendRequest', data: {requestId: 3}}, - {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page2'}}, - - {name: 'ResourceSendRequest', data: {requestId: 4}}, - {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page3'}}, - ]); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - const getDependencyIds = node => node.getDependencies().map(node => node.id); - - assert.equal(nodes.length, 6); - - assert.deepEqual(getDependencyIds(nodes[0]), []); - assert.deepEqual(getDependencyIds(nodes[1]), [0]); - assert.deepEqual(getDependencyIds(nodes[2]), [0]); - assert.deepEqual(getDependencyIds(nodes[3]), [0, '1.220000']); - assert.deepEqual(getDependencyIds(nodes[4]), [0, '1.220000']); - - assert.equal('1.220000', nodes[5].id); - assert.deepEqual(getDependencyIds(nodes[5]), [1, 2]); - }); - - it('should not prune short, first tasks of critical events', () => { - const request0 = createRequest(0, 'https://example.com/page0', 0); - const networkRecords = [request0]; - URL = {requestedUrl: 'https://example.com/page0', mainDocumentUrl: 'https://example.com/page0'}; - - const makeShortEvent = firstEventName => { - const startTs = processedTrace.mainThreadEvents.length * 100; - addTaskEvents(startTs, 5, [ - {name: firstEventName, data: {url: 'https://example.com/page0'}}, - ]); - }; - - const criticalEventNames = [ - 'Paint', - 'Layout', - 'ParseHTML', - ]; - for (const eventName of criticalEventNames) { - makeShortEvent(eventName); - makeShortEvent(eventName); - } - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const cpuNodes = []; - graph.traverse(node => node.type === 'cpu' && cpuNodes.push(node)); - - expect(cpuNodes.map(node => { - return { - id: node.id, - name: node.childEvents[0].name, - }; - })).toEqual([ - { - id: '1.0', - name: 'Paint', - }, - { - // ID jumps by 4 between each because each node has 2 CPU tasks and we skip the 2nd of each event type - id: '1.400000', - name: 'Layout', - }, - { - id: '1.800000', - name: 'ParseHTML', - }, - ]); - }); - - it('should set isMainDocument on request with mainDocumentUrl', () => { - const request1 = createRequest(1, 'https://example.com/', 0, null, NetworkRequest.TYPES.Other); - const request2 = createRequest(2, 'https://example.com/page', 5, null, NetworkRequest.TYPES.Document); - // Add in another unrelated + early request to make sure we pick the correct chain - const request3 = createRequest(3, 'https://example.com/page2', 0, null, NetworkRequest.TYPES.Other); - request2.redirects = [request1]; - const networkRecords = [request1, request2, request3]; - URL = {requestedUrl: 'https://example.com/', mainDocumentUrl: 'https://example.com/page'}; - - addTaskEvents(0, 0, []); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - assert.equal(nodes.length, 3); - assert.equal(nodes[0].id, 1); - assert.equal(nodes[0].isMainDocument(), false); - assert.equal(nodes[1].isMainDocument(), true); - assert.equal(nodes[2].isMainDocument(), false); - }); - - it('should link up script initiators', () => { - const request1 = createRequest(1, 'https://example.com/', 0); - const request2 = createRequest(2, 'https://example.com/page', 5); - const request3 = createRequest(3, 'https://example.com/page2', 5); - const request4 = createRequest(4, 'https://example.com/page3', 20); - // Set multiple initiator requests through script stack. - request4.initiator = { - type: 'script', - stack: {callFrames: [{url: 'https://example.com/page'}], parent: {parent: {callFrames: [{url: 'https://example.com/page2'}]}}}, - }; - // Also set the initiatorRequest that Lighthouse's network-recorder.js creates. - // This should be ignored and only used as a fallback. - request4.initiatorRequest = request1; - const networkRecords = [request1, request2, request3, request4]; - - addTaskEvents(0, 0, []); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - assert.equal(nodes.length, 4); - assert.deepEqual(nodes.map(node => node.id), [1, 2, 3, 4]); - assert.deepEqual(nodes[0].getDependencies(), []); - assert.deepEqual(nodes[1].getDependencies(), [nodes[0]]); - assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); - assert.deepEqual(nodes[3].getDependencies(), [nodes[1], nodes[2]]); - }); - - it('should link up script initiators only when timing is valid', () => { - const request1 = createRequest(1, 'https://example.com/', 0); - const request2 = createRequest(2, 'https://example.com/page', 500); - const request3 = createRequest(3, 'https://example.com/page2', 500); - const request4 = createRequest(4, 'https://example.com/page3', 20); - request4.initiator = { - type: 'script', - stack: {callFrames: [{url: 'https://example.com/page'}], parent: {parent: {callFrames: [{url: 'https://example.com/page2'}]}}}, - }; - const networkRecords = [request1, request2, request3, request4]; - - addTaskEvents(0, 0, []); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - assert.equal(nodes.length, 4); - assert.deepEqual(nodes.map(node => node.id), [1, 2, 3, 4]); - assert.deepEqual(nodes[0].getDependencies(), []); - assert.deepEqual(nodes[1].getDependencies(), [nodes[0]]); - assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); - assert.deepEqual(nodes[3].getDependencies(), [nodes[0]]); - }); - - it('should link up script initiators with prefetch requests', () => { - const request1 = createRequest(1, 'https://a.com/1', 0); - const request2Prefetch = createRequest(2, 'https://a.com/js', 5); - const request2Fetch = createRequest(3, 'https://a.com/js', 10); - const request3 = createRequest(4, 'https://a.com/4', 20); - // Set the initiator to an ambiguous URL (there are 2 requests for https://a.com/js) - request3.initiator = { - type: 'script', - stack: {callFrames: [{url: 'https://a.com/js'}], parent: {parent: {callFrames: [{url: 'js'}]}}}, - }; - // Set the initiatorRequest that it should fallback to. - request3.initiatorRequest = request2Fetch; - const networkRecords = [request1, request2Prefetch, request2Fetch, request3]; - URL = {requestedUrl: 'https://a.com/1', mainDocumentUrl: 'https://a.com/1'}; - - addTaskEvents(0, 0, []); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - assert.equal(nodes.length, 4); - assert.deepEqual(nodes.map(node => node.id), [1, 2, 3, 4]); - assert.deepEqual(nodes[0].getDependencies(), []); - assert.deepEqual(nodes[1].getDependencies(), [nodes[0]]); - assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); - assert.deepEqual(nodes[3].getDependencies(), [nodes[2]]); - }); - - it('should not link up initiators with circular dependencies', () => { - const rootRequest = createRequest(1, 'https://a.com', 0); - // jsRequest1 initiated by jsRequest2 - // *AND* - // jsRequest2 initiated by jsRequest1 - const jsRequest1 = createRequest(2, 'https://a.com/js1', 1, {url: 'https://a.com/js2'}); - const jsRequest2 = createRequest(3, 'https://a.com/js2', 1, {url: 'https://a.com/js1'}); - const networkRecords = [rootRequest, jsRequest1, jsRequest2]; - URL = {requestedUrl: 'https://a.com', mainDocumentUrl: 'https://a.com'}; - - addTaskEvents(0, 0, []); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - nodes.sort((a, b) => a.id - b.id); - - assert.equal(nodes.length, 3); - assert.deepEqual(nodes.map(node => node.id), [1, 2, 3]); - assert.deepEqual(nodes[0].getDependencies(), []); - // We don't know which of the initiators to trust in a cycle, so for now we - // trust the earliest one (mostly because it's simplest). - // In the wild so far we've only seen this for self-referential relationships. - // If the evidence changes, then feel free to change these expectations :) - assert.deepEqual(nodes[1].getDependencies(), [nodes[2]]); - assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); - }); - - it('should not link up initiatorRequests with circular dependencies', () => { - const rootRequest = createRequest(1, 'https://a.com', 0); - // jsRequest1 initiated by jsRequest2 - // *AND* - // jsRequest2 initiated by jsRequest1 - const jsRequest1 = createRequest(2, 'https://a.com/js1', 1); - const jsRequest2 = createRequest(3, 'https://a.com/js2', 1); - jsRequest1.initiatorRequest = jsRequest2; - jsRequest2.initiatorRequest = jsRequest1; - const networkRecords = [rootRequest, jsRequest1, jsRequest2]; - URL = {requestedUrl: 'https://a.com', mainDocumentUrl: 'https://a.com'}; - - addTaskEvents(0, 0, []); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - nodes.sort((a, b) => a.id - b.id); - - assert.equal(nodes.length, 3); - assert.deepEqual(nodes.map(node => node.id), [1, 2, 3]); - assert.deepEqual(nodes[0].getDependencies(), []); - assert.deepEqual(nodes[1].getDependencies(), [nodes[2]]); - assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); - }); - - it('should find root if it is not the first node', () => { - const request1 = createRequest(1, 'https://example.com/', 0, null, NetworkRequest.TYPES.Other); - const request2 = createRequest(2, 'https://example.com/page', 5, null, NetworkRequest.TYPES.Document); - const networkRecords = [request1, request2]; - URL = {requestedUrl: 'https://example.com/page', mainDocumentUrl: 'https://example.com/page'}; - - // Evaluated before root request. - addTaskEvents(0.1, 50, [ - {name: 'EvaluateScript'}, - ]); - - const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); - const nodes = []; - graph.traverse(node => nodes.push(node)); - - assert.equal(nodes.length, 1); - assert.deepEqual(nodes.map(node => node.id), [2]); - assert.deepEqual(nodes[0].getDependencies(), []); - assert.deepEqual(nodes[0].getDependents(), []); - }); - }); }); diff --git a/core/test/lib/lantern/page-dependency-graph-test.js b/core/test/lib/lantern/page-dependency-graph-test.js new file mode 100644 index 000000000000..9bc76930f22b --- /dev/null +++ b/core/test/lib/lantern/page-dependency-graph-test.js @@ -0,0 +1,623 @@ +/** + * @license + * Copyright 2024 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'assert/strict'; + +import {PageDependencyGraph} from '../../../lib/lantern/page-dependency-graph.js'; +import {NetworkRequestTypes} from '../../../lib/lantern/lantern.js'; + +function createRequest( + requestId, + url, + rendererStartTime = 0, + initiator = null, + resourceType = NetworkRequestTypes.Document, + sessionTargetType = 'page' +) { + const networkEndTime = rendererStartTime + 50; + return { + requestId, + url, + rendererStartTime, + networkEndTime, + initiator, + resourceType, + sessionTargetType, + }; +} + +const TOPLEVEL_TASK_NAME = 'TaskQueueManager::ProcessTaskFromWorkQueue'; +describe('PageDependencyGraph computed artifact:', () => { + let processedTrace; + let URL; + + function addTaskEvents(startTs, duration, evts) { + const mainEvent = { + name: TOPLEVEL_TASK_NAME, + tid: 1, + ts: startTs * 1000, + dur: duration * 1000, + args: {}, + }; + + processedTrace.mainThreadEvents.push(mainEvent); + + let i = 0; + for (const evt of evts) { + i++; + processedTrace.mainThreadEvents.push({ + name: evt.name, + ts: (evt.ts * 1000) || (startTs * 1000 + i), + args: {data: evt.data}, + }); + } + } + + beforeEach(() => { + processedTrace = {mainThreadEvents: []}; + URL = {requestedUrl: 'https://example.com/', mainDocumentUrl: 'https://example.com/'}; + }); + + describe('#getNetworkNodeOutput', () => { + const request1 = createRequest(1, 'https://example.com/'); + const request2 = createRequest(2, 'https://example.com/page'); + const request3 = createRequest(3, 'https://example.com/page'); + const networkRecords = [request1, request2, request3]; + + it('should create network nodes', () => { + const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords); + for (let i = 0; i < networkRecords.length; i++) { + const node = networkNodeOutput.nodes[i]; + assert.ok(node, `did not create node at index ${i}`); + assert.equal(node.id, i + 1); + assert.equal(node.type, 'network'); + assert.equal(node.request, networkRecords[i]); + } + }); + + it('should ignore worker requests', () => { + const workerRequest = createRequest(4, 'https://example.com/worker.js', 0, null, 'Script', 'worker'); + const recordsWithWorker = [ + ...networkRecords, + workerRequest, + ]; + + const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(recordsWithWorker); + + expect(networkNodeOutput.nodes).toHaveLength(3); + expect(networkNodeOutput.nodes.map(node => node.request)).not.toContain(workerRequest); + }); + + it('should index nodes by ID', () => { + const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords); + const indexedById = networkNodeOutput.idToNodeMap; + for (const record of networkRecords) { + assert.equal(indexedById.get(record.requestId).request, record); + } + }); + + it('should index nodes by URL', () => { + const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput(networkRecords); + const nodes = networkNodeOutput.nodes; + const indexedByUrl = networkNodeOutput.urlToNodeMap; + assert.deepEqual(indexedByUrl.get('https://example.com/'), [nodes[0]]); + assert.deepEqual(indexedByUrl.get('https://example.com/page'), [nodes[1], nodes[2]]); + }); + + it('should index nodes by frame', () => { + const networkNodeOutput = PageDependencyGraph.getNetworkNodeOutput([ + {...createRequest(1, 'https://example.com/'), documentURL: 'https://example.com/', frameId: 'A'}, + {...createRequest(2, 'https://example.com/page'), documentURL: 'https://example.com/', frameId: 'A'}, + {...createRequest(3, 'https://example.com/page2'), documentURL: 'https://example.com/page2', frameId: 'C', + resourceType: NetworkRequestTypes.XHR}, + {...createRequest(4, 'https://example.com/page3'), documentURL: 'https://example.com/page3', frameId: 'D'}, + {...createRequest(4, 'https://example.com/page4'), documentURL: 'https://example.com/page4', frameId: undefined}, + {...createRequest(4, 'https://example.com/page5'), documentURL: 'https://example.com/page5', frameId: 'collision'}, + {...createRequest(4, 'https://example.com/page6'), documentURL: 'https://example.com/page6', frameId: 'collision'}, + ]); + + const nodes = networkNodeOutput.nodes; + const indexedByFrame = networkNodeOutput.frameIdToNodeMap; + expect([...indexedByFrame.entries()]).toEqual([ + ['A', nodes[0]], + ['D', nodes[3]], + ['collision', null], + ]); + }); + }); + + describe('#getCPUNodes', () => { + it('should create CPU nodes', () => { + addTaskEvents(0, 100, [ + {name: 'MyCustomEvent'}, + {name: 'OtherEvent'}, + {name: 'OutsideTheWindow', ts: 200}, + {name: 'OrphanedEvent'}, // should be ignored since we stopped at OutsideTheWindow + ]); + + addTaskEvents(250, 50, [ + {name: 'LaterEvent'}, + ]); + + assert.equal(processedTrace.mainThreadEvents.length, 7); + const nodes = PageDependencyGraph.getCPUNodes(processedTrace); + assert.equal(nodes.length, 2); + + const node1 = nodes[0]; + assert.equal(node1.id, '1.0'); + assert.equal(node1.type, 'cpu'); + assert.equal(node1.event, processedTrace.mainThreadEvents[0]); + assert.equal(node1.childEvents.length, 2); + assert.equal(node1.childEvents[1].name, 'OtherEvent'); + + const node2 = nodes[1]; + assert.equal(node2.id, '1.250000'); + assert.equal(node2.type, 'cpu'); + assert.equal(node2.event, processedTrace.mainThreadEvents[5]); + assert.equal(node2.childEvents.length, 1); + assert.equal(node2.childEvents[0].name, 'LaterEvent'); + }); + }); + + describe('#createGraph', () => { + it('should compute a simple network graph', () => { + const request1 = createRequest(1, 'https://example.com/', 0); + const request2 = createRequest(2, 'https://example.com/page', 5); + const request3 = createRequest(3, 'https://example.com/page2', 5); + const request4 = createRequest(4, 'https://example.com/page3', 10, {url: 'https://example.com/page'}); + const networkRecords = [request1, request2, request3, request4]; + + addTaskEvents(0, 0, []); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + assert.equal(nodes.length, 4); + assert.deepEqual(nodes.map(node => node.id), [1, 2, 3, 4]); + assert.deepEqual(nodes[0].getDependencies(), []); + assert.deepEqual(nodes[1].getDependencies(), [nodes[0]]); + assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); + assert.deepEqual(nodes[3].getDependencies(), [nodes[1]]); + }); + + it('should compute a simple network and CPU graph', () => { + const request1 = createRequest(1, 'https://example.com/', 0); + const request2 = createRequest(2, 'https://example.com/page', 50); + const request3 = createRequest(3, 'https://example.com/page2', 50); + const request4 = createRequest(4, 'https://example.com/page3', 300, null, NetworkRequestTypes.XHR); + const networkRecords = [request1, request2, request3, request4]; + + addTaskEvents(200, 200, [ + {name: 'EvaluateScript', data: {url: 'https://example.com/page'}}, + {name: 'ResourceSendRequest', data: {requestId: 4}}, + ]); + + addTaskEvents(700, 50, [ + {name: 'InvalidateLayout', data: {stackTrace: [{url: 'https://example.com/page2'}]}}, + {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page3'}}, + ]); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + const getIds = nodes => nodes.map(node => node.id); + const getDependencyIds = node => getIds(node.getDependencies()); + + assert.equal(nodes.length, 6); + assert.deepEqual(getIds(nodes), [1, 2, 3, 4, '1.200000', '1.700000']); + assert.deepEqual(getDependencyIds(nodes[0]), []); + assert.deepEqual(getDependencyIds(nodes[1]), [1]); + assert.deepEqual(getDependencyIds(nodes[2]), [1]); + assert.deepEqual(getDependencyIds(nodes[3]), [1, '1.200000']); + assert.deepEqual(getDependencyIds(nodes[4]), [2]); + assert.deepEqual(getDependencyIds(nodes[5]), [3, 4]); + }); + + it('should compute a network graph with duplicate URLs', () => { + const request1 = createRequest(1, 'https://example.com/', 0); + const request2 = createRequest(2, 'https://example.com/page', 5); + const request3 = createRequest(3, 'https://example.com/page', 5); // duplicate URL + const request4 = createRequest(4, 'https://example.com/page3', 10, {url: 'https://example.com/page'}); + const networkRecords = [request1, request2, request3, request4]; + + addTaskEvents(0, 0, []); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + assert.equal(nodes.length, 4); + assert.deepEqual(nodes.map(node => node.id), [1, 2, 3, 4]); + assert.deepEqual(nodes[0].getDependencies(), []); + assert.deepEqual(nodes[1].getDependencies(), [nodes[0]]); + assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); + assert.deepEqual(nodes[3].getDependencies(), [nodes[0]]); // should depend on rootNode instead + }); + + it('should be forgiving without cyclic dependencies', () => { + const request1 = createRequest(1, 'https://example.com/', 0); + const request2 = createRequest(2, 'https://example.com/page', 250, null, NetworkRequestTypes.XHR); + const request3 = createRequest(3, 'https://example.com/page2', 210); + const request4 = createRequest(4, 'https://example.com/page3', 590); + const request5 = createRequest(5, 'https://example.com/page4', 595, null, NetworkRequestTypes.XHR); + const networkRecords = [request1, request2, request3, request4, request5]; + + addTaskEvents(200, 200, [ + // CPU 1.2 should depend on Network 1 + {name: 'EvaluateScript', data: {url: 'https://example.com/'}}, + + // Network 2 should depend on CPU 1.2, but 1.2 should not depend on Network 1 + {name: 'ResourceSendRequest', data: {requestId: 2}}, + {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page'}}, + + // CPU 1.2 should not depend on Network 3 because it starts after CPU 1.2 + {name: 'EvaluateScript', data: {url: 'https://example.com/page2'}}, + ]); + + addTaskEvents(600, 150, [ + // CPU 1.6 should depend on Network 4 even though it ends at 410ms + {name: 'InvalidateLayout', data: {stackTrace: [{url: 'https://example.com/page3'}]}}, + // Network 5 should not depend on CPU 1.6 because it started before CPU 1.6 + {name: 'ResourceSendRequest', data: {requestId: 5}}, + ]); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + const getDependencyIds = node => node.getDependencies().map(node => node.id); + + assert.equal(nodes.length, 7); + assert.deepEqual(getDependencyIds(nodes[0]), []); + assert.deepEqual(getDependencyIds(nodes[1]), [1, '1.200000']); + assert.deepEqual(getDependencyIds(nodes[2]), [1]); + assert.deepEqual(getDependencyIds(nodes[3]), [1]); + assert.deepEqual(getDependencyIds(nodes[4]), [1]); + assert.deepEqual(getDependencyIds(nodes[5]), [1]); + assert.deepEqual(getDependencyIds(nodes[6]), [4]); + }); + + it('should not install timer dependency on itself', () => { + const request1 = createRequest(1, 'https://example.com/', 0); + const networkRecords = [request1]; + + addTaskEvents(200, 200, [ + // CPU 1.2 should depend on Network 1 + {name: 'EvaluateScript', data: {url: 'https://example.com/'}}, + // CPU 1.2 will install and fire it's own timer, but should not depend on itself + {name: 'TimerInstall', data: {timerId: 'timer1'}}, + {name: 'TimerFire', data: {timerId: 'timer1'}}, + ]); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + const getDependencyIds = node => node.getDependencies().map(node => node.id); + + assert.equal(nodes.length, 2); + assert.deepEqual(getDependencyIds(nodes[0]), []); + assert.deepEqual(getDependencyIds(nodes[1]), [1]); + }); + + it('should prune short tasks', () => { + const request0 = createRequest(0, 'https://example.com/page0', 0); + const request1 = createRequest(1, 'https://example.com/', 100, null, NetworkRequestTypes.Script); + const request2 = createRequest(2, 'https://example.com/page', 200, null, NetworkRequestTypes.XHR); + const request3 = createRequest(3, 'https://example.com/page2', 300, null, NetworkRequestTypes.Script); + const request4 = createRequest(4, 'https://example.com/page3', 400, null, NetworkRequestTypes.XHR); + const networkRecords = [request0, request1, request2, request3, request4]; + URL = {requestedUrl: 'https://example.com/page0', mainDocumentUrl: 'https://example.com/page0'}; + + // Long task, should be kept in the output. + addTaskEvents(120, 50, [ + {name: 'EvaluateScript', data: {url: 'https://example.com/'}}, + {name: 'ResourceSendRequest', data: {requestId: 2}}, + {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page'}}, + ]); + + // Short task, should be pruned, but the 3->4 relationship should be retained + addTaskEvents(350, 5, [ + {name: 'EvaluateScript', data: {url: 'https://example.com/page2'}}, + {name: 'ResourceSendRequest', data: {requestId: 4}}, + {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page3'}}, + ]); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + const getDependencyIds = node => node.getDependencies().map(node => node.id); + + assert.equal(nodes.length, 6); + + assert.deepEqual(getDependencyIds(nodes[0]), []); + assert.deepEqual(getDependencyIds(nodes[1]), [0]); + assert.deepEqual(getDependencyIds(nodes[2]), [0, '1.120000']); + assert.deepEqual(getDependencyIds(nodes[3]), [0]); + assert.deepEqual(getDependencyIds(nodes[4]), [0, 3]); + + assert.equal('1.120000', nodes[5].id); + assert.deepEqual(getDependencyIds(nodes[5]), [1]); + }); + + it('should not prune highly-connected short tasks', () => { + const request0 = createRequest(0, 'https://example.com/page0', 0); + const request1 = { + ...createRequest(1, 'https://example.com/', 100, null, NetworkRequestTypes.Document), + documentURL: 'https://example.com/', + frameId: 'frame1', + }; + const request2 = { + ...createRequest(2, 'https://example.com/page', 200, null, NetworkRequestTypes.Script), + documentURL: 'https://example.com/', + frameId: 'frame1', + }; + const request3 = createRequest(3, 'https://example.com/page2', 300, null, NetworkRequestTypes.XHR); + const request4 = createRequest(4, 'https://example.com/page3', 400, null, NetworkRequestTypes.XHR); + const networkRecords = [request0, request1, request2, request3, request4]; + URL = {requestedUrl: 'https://example.com/page0', mainDocumentUrl: 'https://example.com/page0'}; + + // Short task, evaluates script (2) and sends two XHRs. + addTaskEvents(220, 5, [ + {name: 'EvaluateScript', data: {url: 'https://example.com/page', frame: 'frame1'}}, + + {name: 'ResourceSendRequest', data: {requestId: 3}}, + {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page2'}}, + + {name: 'ResourceSendRequest', data: {requestId: 4}}, + {name: 'XHRReadyStateChange', data: {readyState: 4, url: 'https://example.com/page3'}}, + ]); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + const getDependencyIds = node => node.getDependencies().map(node => node.id); + + assert.equal(nodes.length, 6); + + assert.deepEqual(getDependencyIds(nodes[0]), []); + assert.deepEqual(getDependencyIds(nodes[1]), [0]); + assert.deepEqual(getDependencyIds(nodes[2]), [0]); + assert.deepEqual(getDependencyIds(nodes[3]), [0, '1.220000']); + assert.deepEqual(getDependencyIds(nodes[4]), [0, '1.220000']); + + assert.equal('1.220000', nodes[5].id); + assert.deepEqual(getDependencyIds(nodes[5]), [1, 2]); + }); + + it('should not prune short, first tasks of critical events', () => { + const request0 = createRequest(0, 'https://example.com/page0', 0); + const networkRecords = [request0]; + URL = {requestedUrl: 'https://example.com/page0', mainDocumentUrl: 'https://example.com/page0'}; + + const makeShortEvent = firstEventName => { + const startTs = processedTrace.mainThreadEvents.length * 100; + addTaskEvents(startTs, 5, [ + {name: firstEventName, data: {url: 'https://example.com/page0'}}, + ]); + }; + + const criticalEventNames = [ + 'Paint', + 'Layout', + 'ParseHTML', + ]; + for (const eventName of criticalEventNames) { + makeShortEvent(eventName); + makeShortEvent(eventName); + } + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const cpuNodes = []; + graph.traverse(node => node.type === 'cpu' && cpuNodes.push(node)); + + expect(cpuNodes.map(node => { + return { + id: node.id, + name: node.childEvents[0].name, + }; + })).toEqual([ + { + id: '1.0', + name: 'Paint', + }, + { + // ID jumps by 4 between each because each node has 2 CPU tasks and we skip the 2nd of each event type + id: '1.400000', + name: 'Layout', + }, + { + id: '1.800000', + name: 'ParseHTML', + }, + ]); + }); + + it('should set isMainDocument on request with mainDocumentUrl', () => { + const request1 = createRequest(1, 'https://example.com/', 0, null, NetworkRequestTypes.Other); + const request2 = createRequest(2, 'https://example.com/page', 5, null, NetworkRequestTypes.Document); + // Add in another unrelated + early request to make sure we pick the correct chain + const request3 = createRequest(3, 'https://example.com/page2', 0, null, NetworkRequestTypes.Other); + request2.redirects = [request1]; + const networkRecords = [request1, request2, request3]; + URL = {requestedUrl: 'https://example.com/', mainDocumentUrl: 'https://example.com/page'}; + + addTaskEvents(0, 0, []); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + assert.equal(nodes.length, 3); + assert.equal(nodes[0].id, 1); + assert.equal(nodes[0].isMainDocument(), false); + assert.equal(nodes[1].isMainDocument(), true); + assert.equal(nodes[2].isMainDocument(), false); + }); + + it('should link up script initiators', () => { + const request1 = createRequest(1, 'https://example.com/', 0); + const request2 = createRequest(2, 'https://example.com/page', 5); + const request3 = createRequest(3, 'https://example.com/page2', 5); + const request4 = createRequest(4, 'https://example.com/page3', 20); + // Set multiple initiator requests through script stack. + request4.initiator = { + type: 'script', + stack: {callFrames: [{url: 'https://example.com/page'}], parent: {parent: {callFrames: [{url: 'https://example.com/page2'}]}}}, + }; + // Also set the initiatorRequest that Lighthouse's network-recorder.js creates. + // This should be ignored and only used as a fallback. + request4.initiatorRequest = request1; + const networkRecords = [request1, request2, request3, request4]; + + addTaskEvents(0, 0, []); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + assert.equal(nodes.length, 4); + assert.deepEqual(nodes.map(node => node.id), [1, 2, 3, 4]); + assert.deepEqual(nodes[0].getDependencies(), []); + assert.deepEqual(nodes[1].getDependencies(), [nodes[0]]); + assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); + assert.deepEqual(nodes[3].getDependencies(), [nodes[1], nodes[2]]); + }); + + it('should link up script initiators only when timing is valid', () => { + const request1 = createRequest(1, 'https://example.com/', 0); + const request2 = createRequest(2, 'https://example.com/page', 500); + const request3 = createRequest(3, 'https://example.com/page2', 500); + const request4 = createRequest(4, 'https://example.com/page3', 20); + request4.initiator = { + type: 'script', + stack: {callFrames: [{url: 'https://example.com/page'}], parent: {parent: {callFrames: [{url: 'https://example.com/page2'}]}}}, + }; + const networkRecords = [request1, request2, request3, request4]; + + addTaskEvents(0, 0, []); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + assert.equal(nodes.length, 4); + assert.deepEqual(nodes.map(node => node.id), [1, 2, 3, 4]); + assert.deepEqual(nodes[0].getDependencies(), []); + assert.deepEqual(nodes[1].getDependencies(), [nodes[0]]); + assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); + assert.deepEqual(nodes[3].getDependencies(), [nodes[0]]); + }); + + it('should link up script initiators with prefetch requests', () => { + const request1 = createRequest(1, 'https://a.com/1', 0); + const request2Prefetch = createRequest(2, 'https://a.com/js', 5); + const request2Fetch = createRequest(3, 'https://a.com/js', 10); + const request3 = createRequest(4, 'https://a.com/4', 20); + // Set the initiator to an ambiguous URL (there are 2 requests for https://a.com/js) + request3.initiator = { + type: 'script', + stack: {callFrames: [{url: 'https://a.com/js'}], parent: {parent: {callFrames: [{url: 'js'}]}}}, + }; + // Set the initiatorRequest that it should fallback to. + request3.initiatorRequest = request2Fetch; + const networkRecords = [request1, request2Prefetch, request2Fetch, request3]; + URL = {requestedUrl: 'https://a.com/1', mainDocumentUrl: 'https://a.com/1'}; + + addTaskEvents(0, 0, []); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + assert.equal(nodes.length, 4); + assert.deepEqual(nodes.map(node => node.id), [1, 2, 3, 4]); + assert.deepEqual(nodes[0].getDependencies(), []); + assert.deepEqual(nodes[1].getDependencies(), [nodes[0]]); + assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); + assert.deepEqual(nodes[3].getDependencies(), [nodes[2]]); + }); + + it('should not link up initiators with circular dependencies', () => { + const rootRequest = createRequest(1, 'https://a.com', 0); + // jsRequest1 initiated by jsRequest2 + // *AND* + // jsRequest2 initiated by jsRequest1 + const jsRequest1 = createRequest(2, 'https://a.com/js1', 1, {url: 'https://a.com/js2'}); + const jsRequest2 = createRequest(3, 'https://a.com/js2', 1, {url: 'https://a.com/js1'}); + const networkRecords = [rootRequest, jsRequest1, jsRequest2]; + URL = {requestedUrl: 'https://a.com', mainDocumentUrl: 'https://a.com'}; + + addTaskEvents(0, 0, []); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + nodes.sort((a, b) => a.id - b.id); + + assert.equal(nodes.length, 3); + assert.deepEqual(nodes.map(node => node.id), [1, 2, 3]); + assert.deepEqual(nodes[0].getDependencies(), []); + // We don't know which of the initiators to trust in a cycle, so for now we + // trust the earliest one (mostly because it's simplest). + // In the wild so far we've only seen this for self-referential relationships. + // If the evidence changes, then feel free to change these expectations :) + assert.deepEqual(nodes[1].getDependencies(), [nodes[2]]); + assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); + }); + + it('should not link up initiatorRequests with circular dependencies', () => { + const rootRequest = createRequest(1, 'https://a.com', 0); + // jsRequest1 initiated by jsRequest2 + // *AND* + // jsRequest2 initiated by jsRequest1 + const jsRequest1 = createRequest(2, 'https://a.com/js1', 1); + const jsRequest2 = createRequest(3, 'https://a.com/js2', 1); + jsRequest1.initiatorRequest = jsRequest2; + jsRequest2.initiatorRequest = jsRequest1; + const networkRecords = [rootRequest, jsRequest1, jsRequest2]; + URL = {requestedUrl: 'https://a.com', mainDocumentUrl: 'https://a.com'}; + + addTaskEvents(0, 0, []); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + nodes.sort((a, b) => a.id - b.id); + + assert.equal(nodes.length, 3); + assert.deepEqual(nodes.map(node => node.id), [1, 2, 3]); + assert.deepEqual(nodes[0].getDependencies(), []); + assert.deepEqual(nodes[1].getDependencies(), [nodes[2]]); + assert.deepEqual(nodes[2].getDependencies(), [nodes[0]]); + }); + + it('should find root if it is not the first node', () => { + const request1 = createRequest(1, 'https://example.com/', 0, null, NetworkRequestTypes.Other); + const request2 = createRequest(2, 'https://example.com/page', 5, null, NetworkRequestTypes.Document); + const networkRecords = [request1, request2]; + URL = {requestedUrl: 'https://example.com/page', mainDocumentUrl: 'https://example.com/page'}; + + // Evaluated before root request. + addTaskEvents(0.1, 50, [ + {name: 'EvaluateScript'}, + ]); + + const graph = PageDependencyGraph.createGraph(processedTrace, networkRecords, URL); + const nodes = []; + graph.traverse(node => nodes.push(node)); + + assert.equal(nodes.length, 1); + assert.deepEqual(nodes.map(node => node.id), [2]); + assert.deepEqual(nodes[0].getDependencies(), []); + assert.deepEqual(nodes[0].getDependents(), []); + }); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 15df20fa4a78..31c016da62f4 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -69,6 +69,7 @@ "core/test/lib/i18n/i18n-test.js", "core/test/lib/icons-test.js", "core/test/lib/lantern/base-node-test.js", + "core/test/lib/lantern/page-dependency-graph-test.js", "core/test/lib/lantern/simulator/connection-pool-test.js", "core/test/lib/lantern/simulator/dns-cache-test.js", "core/test/lib/lantern/simulator/network-analyzer-test.js", diff --git a/types/internal/lantern.d.ts b/types/internal/lantern.d.ts index 54b07fc83c35..9581ada830cf 100644 --- a/types/internal/lantern.d.ts +++ b/types/internal/lantern.d.ts @@ -51,6 +51,7 @@ declare namespace Lantern { url: string; protocol: string; parsedURL: ParsedURL; + documentURL: string; /** When the renderer process initially discovers a network request, in milliseconds. */ rendererStartTime: number; /** @@ -75,9 +76,15 @@ declare namespace Lantern { redirectDestination: NetworkRequest | undefined; failed: boolean; initiator: LH.Crdp.Network.Initiator; + initiatorRequest: NetworkRequest | undefined; + /** The chain of network requests that redirected to this one */ + redirects: NetworkRequest[] | undefined; timing: LH.Crdp.Network.ResourceTiming | undefined; resourceType: LH.Crdp.Network.ResourceType | undefined; + mimeType: string; priority: LH.Crdp.Network.ResourcePriority; + frameId: string | undefined; + sessionTargetType: LH.Protocol.TargetType | undefined; } namespace Simulation {