core(fr): user triggered navigations

lighthouse-cli/run.js

@@ -208,7 +208,7 @@ async function runLighthouseWithFraggleRock(url, flags, config, launchedChrome)
   const page = await browser.newPage();
   flags.channel = 'fraggle-rock-cli';
   const configContext = {configPath: flags.configPath, settingsOverrides: flags};
-  return fraggleRock.navigation({url, page, config, configContext});
+  return fraggleRock.navigation({requestor: url, page, config, configContext});
 }
 
 /**

lighthouse-core/fraggle-rock/gather/navigation-runner.js

@@ -35,7 +35,7 @@ const NetworkRecords = require('../../computed/network-records.js');
  * @property {Driver} driver
  * @property {LH.Config.FRConfig} config
  * @property {LH.Config.NavigationDefn} navigation
- * @property {string} requestedUrl
+ * @property {LH.NavigationRequestor} requestor
  * @property {LH.FRBaseArtifacts} baseArtifacts
  * @property {Map<string, LH.ArbitraryEqualityMap>} computedCache
  * @property {InternalOptions} [options]
@@ -44,17 +44,17 @@ const NetworkRecords = require('../../computed/network-records.js');
 /** @typedef {Omit<Parameters<typeof collectPhaseArtifacts>[0], 'phase'>} PhaseState */
 
 /**
- * @param {{driver: Driver, config: LH.Config.FRConfig, requestedUrl: string, options?: InternalOptions}} args
+ * @param {{driver: Driver, config: LH.Config.FRConfig, requestor: LH.NavigationRequestor, options?: InternalOptions}} args
  * @return {Promise<{baseArtifacts: LH.FRBaseArtifacts}>}
  */
-async function _setup({driver, config, requestedUrl, options}) {
+async function _setup({driver, config, requestor, options}) {
   await driver.connect();
   if (!options?.skipAboutBlank) {
     await gotoURL(driver, defaultNavigationConfig.blankPage, {waitUntil: ['navigated']});
   }
 
   const baseArtifacts = await getBaseArtifacts(config, driver, {gatherMode: 'navigation'});
-  baseArtifacts.URL.requestedUrl = requestedUrl;
+  if (typeof requestor === 'string') baseArtifacts.URL.requestedUrl = requestor;
 
   await prepare.prepareTargetForNavigationMode(driver, config.settings);
 
@@ -65,7 +65,7 @@ async function _setup({driver, config, requestedUrl, options}) {
  * @param {NavigationContext} navigationContext
  * @return {Promise<{warnings: Array<LH.IcuMessage>}>}
  */
-async function _setupNavigation({requestedUrl, driver, navigation, config, options}) {
+async function _setupNavigation({requestor, driver, navigation, config, options}) {
   if (!options?.skipAboutBlank) {
     await gotoURL(driver, navigation.blankPage, {...navigation, waitUntil: ['navigated']});
   }
@@ -74,7 +74,7 @@ async function _setupNavigation({requestedUrl, driver, navigation, config, optio
     config.settings,
     {
       ...navigation,
-      url: requestedUrl,
+      requestor,
     }
   );
 
@@ -90,25 +90,25 @@ async function _cleanupNavigation({driver}) {
 
 /**
  * @param {NavigationContext} navigationContext
- * @return {Promise<{finalUrl: string, navigationError: LH.LighthouseError | undefined, warnings: Array<LH.IcuMessage>}>}
+ * @return {Promise<{requestedUrl: string, finalUrl: string, navigationError: LH.LighthouseError | undefined, warnings: Array<LH.IcuMessage>}>}
  */
 async function _navigate(navigationContext) {
-  const {driver, config, requestedUrl} = navigationContext;
+  const {driver, config, requestor} = navigationContext;
 
   try {
-    const {finalUrl, warnings} = await gotoURL(driver, requestedUrl, {
+    const {requestedUrl, finalUrl, warnings} = await gotoURL(driver, requestor, {
       ...navigationContext.navigation,
       debugNavigation: config.settings.debugNavigation,
       maxWaitForFcp: config.settings.maxWaitForFcp,
       maxWaitForLoad: config.settings.maxWaitForLoad,
       waitUntil: navigationContext.navigation.pauseAfterFcpMs ? ['fcp', 'load'] : ['load'],
     });
-
-    return {finalUrl, navigationError: undefined, warnings};
+    return {requestedUrl, finalUrl, navigationError: undefined, warnings};
   } catch (err) {
     if (!(err instanceof LighthouseError)) throw err;
     if (err.code !== 'NO_FCP' && err.code !== 'PAGE_HUNG') throw err;
-    return {finalUrl: requestedUrl, navigationError: err, warnings: []};
+    if (typeof requestor !== 'string') throw err;
+    return {requestedUrl: requestor, finalUrl: requestor, navigationError: err, warnings: []};
   }
 }
 
@@ -152,7 +152,7 @@ async function _collectDebugData(navigationContext, phaseState) {
  * @param {PhaseState} phaseState
  * @param {Awaited<ReturnType<typeof _setupNavigation>>} setupResult
  * @param {Awaited<ReturnType<typeof _navigate>>} navigateResult
- * @return {Promise<{finalUrl: string, artifacts: Partial<LH.GathererArtifacts>, warnings: Array<LH.IcuMessage>, pageLoadError: LH.LighthouseError | undefined}>}
+ * @return {Promise<{requestedUrl: string, finalUrl: string, artifacts: Partial<LH.GathererArtifacts>, warnings: Array<LH.IcuMessage>, pageLoadError: LH.LighthouseError | undefined}>}
  */
 async function _computeNavigationResult(
   navigationContext,
@@ -174,7 +174,7 @@ async function _computeNavigationResult(
   if (pageLoadError) {
     const locale = navigationContext.config.settings.locale;
     const localizedMessage = format.getFormatted(pageLoadError.friendlyMessage, locale);
-    log.error('NavigationRunner', localizedMessage, navigationContext.requestedUrl);
+    log.error('NavigationRunner', localizedMessage, navigateResult.requestedUrl);
 
     /** @type {Partial<LH.GathererArtifacts>} */
     const artifacts = {};
@@ -183,6 +183,7 @@ async function _computeNavigationResult(
     if (debugData.trace) artifacts.traces = {[pageLoadErrorId]: debugData.trace};
 
     return {
+      requestedUrl: navigateResult.requestedUrl,
       finalUrl,
       pageLoadError,
       artifacts,
@@ -192,7 +193,13 @@ async function _computeNavigationResult(
     await collectPhaseArtifacts({phase: 'getArtifact', ...phaseState});
 
     const artifacts = await awaitArtifacts(phaseState.artifactState);
-    return {finalUrl, artifacts, warnings, pageLoadError: undefined};
+    return {
+      requestedUrl: navigateResult.requestedUrl,
+      finalUrl,
+      artifacts,
+      warnings,
+      pageLoadError: undefined,
+    };
   }
 }
 
@@ -224,10 +231,10 @@ async function _navigation(navigationContext) {
 }
 
 /**
- * @param {{driver: Driver, config: LH.Config.FRConfig, requestedUrl: string; baseArtifacts: LH.FRBaseArtifacts, computedCache: NavigationContext['computedCache'], options?: InternalOptions}} args
+ * @param {{driver: Driver, config: LH.Config.FRConfig, requestor: LH.NavigationRequestor; baseArtifacts: LH.FRBaseArtifacts, computedCache: NavigationContext['computedCache'], options?: InternalOptions}} args
  * @return {Promise<{artifacts: Partial<LH.FRArtifacts & LH.FRBaseArtifacts>}>}
  */
-async function _navigations({driver, config, requestedUrl, baseArtifacts, computedCache, options}) {
+async function _navigations({driver, config, requestor, baseArtifacts, computedCache, options}) {
   if (!config.navigations) throw new Error('No navigations configured');
 
   /** @type {Partial<LH.FRArtifacts & LH.FRBaseArtifacts>} */
@@ -239,7 +246,7 @@ async function _navigations({driver, config, requestedUrl, baseArtifacts, comput
     const navigationContext = {
       driver,
       navigation,
-      requestedUrl,
+      requestor,
       config,
       baseArtifacts,
       computedCache,
@@ -254,7 +261,10 @@ async function _navigations({driver, config, requestedUrl, baseArtifacts, comput
         shouldHaltNavigations = true;
       }
 
-      artifacts.URL = {requestedUrl, finalUrl: navigationResult.finalUrl};
+      artifacts.URL = {
+        requestedUrl: navigationResult.requestedUrl,
+        finalUrl: navigationResult.finalUrl,
+      };
     }
 
     LighthouseRunWarnings.push(...navigationResult.warnings);
@@ -266,21 +276,21 @@ async function _navigations({driver, config, requestedUrl, baseArtifacts, comput
 }
 
 /**
- * @param {{requestedUrl: string, driver: Driver, config: LH.Config.FRConfig}} args
+ * @param {{requestedUrl?: string, driver: Driver, config: LH.Config.FRConfig}} args
  */
 async function _cleanup({requestedUrl, driver, config}) {
-  const didResetStorage = !config.settings.disableStorageReset;
+  const didResetStorage = !config.settings.disableStorageReset && requestedUrl;
   if (didResetStorage) await storage.clearDataForOrigin(driver.defaultSession, requestedUrl);
 
   await driver.disconnect();
 }
 
 /**
- * @param {{url: string, page: import('puppeteer').Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
+ * @param {{requestor: LH.NavigationRequestor, page: import('puppeteer').Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
  * @return {Promise<LH.RunnerResult|undefined>}
  */
 async function navigation(options) {
-  const {url, page, configContext = {}} = options;
+  const {requestor, page, configContext = {}} = options;
   const {config} = initializeConfig(options.config, {...configContext, gatherMode: 'navigation'});
   const computedCache = new Map();
   const internalOptions = {
@@ -291,8 +301,12 @@ async function navigation(options) {
   const artifacts = await Runner.gather(
     async () => {
       const driver = new Driver(page);
-      const requestedUrl = URL.normalizeUrl(url);
-      const context = {driver, config, requestedUrl, options: internalOptions};
+      const context = {
+        driver,
+        config,
+        requestor: typeof requestor === 'string' ? URL.normalizeUrl(requestor) : requestor,
+        options: internalOptions,
+      };
       const {baseArtifacts} = await _setup(context);
       const {artifacts} = await _navigations({...context, baseArtifacts, computedCache});
       await _cleanup(context);

lighthouse-core/fraggle-rock/user-flow.js

@@ -54,11 +54,11 @@ class UserFlow {
   }
 
   /**
-   * @param {string} url
+   * @param {LH.NavigationRequestor} requestor
    * @param {StepOptions=} stepOptions
    */
-  _getNextNavigationOptions(url, stepOptions) {
-    const options = {url, ...this.options, ...stepOptions};
+  _getNextNavigationOptions(requestor, stepOptions) {
+    const options = {requestor, ...this.options, ...stepOptions};
     const configContext = {...options.configContext};
     const settingsOverrides = {...configContext.settingsOverrides};
 
@@ -81,13 +81,13 @@ class UserFlow {
   }
 
   /**
-   * @param {string} url
+   * @param {LH.NavigationRequestor} requestor
    * @param {StepOptions=} stepOptions
    */
-  async navigate(url, stepOptions) {
+  async navigate(requestor, stepOptions) {
     if (this.currentTimespan) throw Error('Timespan already in progress');
 
-    const result = await navigation(this._getNextNavigationOptions(url, stepOptions));
+    const result = await navigation(this._getNextNavigationOptions(requestor, stepOptions));
     if (!result) throw Error('Navigation returned undefined');
 
     const providedName = stepOptions?.stepName;

lighthouse-core/gather/driver/navigation.js

@@ -78,12 +78,14 @@ function resolveWaitForFullyLoadedOptions(options) {
  * navigations.
  *
  * @param {LH.Gatherer.FRTransitionalDriver} driver
- * @param {string} url
+ * @param {LH.NavigationRequestor} requestor
  * @param {NavigationOptions} options
- * @return {Promise<{finalUrl: string, warnings: Array<LH.IcuMessage>}>}
+ * @return {Promise<{requestedUrl: string, finalUrl: string, warnings: Array<LH.IcuMessage>}>}
  */
-async function gotoURL(driver, url, options) {
-  const status = {msg: `Navigating to ${url}`, id: 'lh:driver:navigate'};
+async function gotoURL(driver, requestor, options) {
+  const status = typeof requestor === 'string' ?
+    {msg: `Navigating to ${requestor}`, id: 'lh:driver:navigate'} :
+    {msg: 'Navigating using a user defined function', id: 'lh:driver:navigate'};
   log.time(status);
 
   const session = driver.defaultSession;
@@ -94,9 +96,14 @@ async function gotoURL(driver, url, options) {
   await session.sendCommand('Page.enable');
   await session.sendCommand('Page.setLifecycleEventsEnabled', {enabled: true});
 
-  // No timeout needed for Page.navigate. See https://github.com/GoogleChrome/lighthouse/pull/6413
-  session.setNextProtocolTimeout(Infinity);
-  const waitforPageNavigateCmd = session.sendCommand('Page.navigate', {url});
+  let waitForPageNavigateCmd;
+  if (typeof requestor === 'string') {
+    // No timeout needed for Page.navigate. See https://github.com/GoogleChrome/lighthouse/pull/6413
+    session.setNextProtocolTimeout(Infinity);
+    waitForPageNavigateCmd = session.sendCommand('Page.navigate', {url: requestor});
+  } else {
+    waitForPageNavigateCmd = requestor();
+  }
 
   const waitForNavigated = options.waitUntil.includes('navigated');
   const waitForLoad = options.waitUntil.includes('load');
@@ -119,10 +126,13 @@ async function gotoURL(driver, url, options) {
 
   const waitConditions = await Promise.all(waitConditionPromises);
   const timedOut = waitConditions.some(condition => condition.timedOut);
-  const finalUrl = (await networkMonitor.getFinalNavigationUrl()) || url;
+  const navigationUrls = await networkMonitor.getNavigationUrls();
+  const requestedUrl = typeof requestor === 'string' ? requestor : navigationUrls.requestedUrl;
+  if (!requestedUrl) throw Error('No navigations detected when running user defined requestor.');
+  const finalUrl = navigationUrls.finalUrl || requestedUrl;
 
   // Bring `Page.navigate` errors back into the promise chain. See https://github.com/GoogleChrome/lighthouse/pull/6739.
-  await waitforPageNavigateCmd;
+  await waitForPageNavigateCmd;
   await networkMonitor.disable();
 
   if (options.debugNavigation) {
@@ -131,8 +141,9 @@ async function gotoURL(driver, url, options) {
 
   log.timeEnd(status);
   return {
+    requestedUrl,
     finalUrl,
-    warnings: getNavigationWarnings({timedOut, finalUrl, requestedUrl: url}),
+    warnings: getNavigationWarnings({timedOut, finalUrl, requestedUrl}),
   };
 }
 

lighthouse-core/gather/driver/network-monitor.js

@@ -133,18 +133,20 @@ class NetworkMonitor {
     this._sessions = new Map();
   }
 
-  /** @return {Promise<string | undefined>} */
-  async getFinalNavigationUrl() {
+  /** @return {Promise<{requestedUrl?: string, finalUrl?: string}>} */
+  async getNavigationUrls() {
     const frameNavigations = this._frameNavigations;
-    if (!frameNavigations.length) return undefined;
+    if (!frameNavigations.length) return {};
 
     const resourceTreeResponse = await this._session.sendCommand('Page.getResourceTree');
     const mainFrameId = resourceTreeResponse.frameTree.frame.id;
     const mainFrameNavigations = frameNavigations.filter(frame => frame.id === mainFrameId);
-    const finalNavigation = mainFrameNavigations[mainFrameNavigations.length - 1];
-    if (!finalNavigation) log.warn('NetworkMonitor', 'No detected navigations');
+    if (!mainFrameNavigations.length) log.warn('NetworkMonitor', 'No detected navigations');
 
-    return finalNavigation?.url;
+    return {
+      requestedUrl: mainFrameNavigations[0]?.url,
+      finalUrl: mainFrameNavigations[mainFrameNavigations.length - 1]?.url,
+    };
   }
 
   /**

lighthouse-core/gather/driver/prepare.js

@@ -71,17 +71,17 @@ async function dismissJavaScriptDialogs(session) {
 
 /**
  * @param {LH.Gatherer.FRProtocolSession} session
- * @param {{url: string}} navigation
+ * @param {string} url
  * @return {Promise<{warnings: Array<LH.IcuMessage>}>}
  */
-async function resetStorageForNavigation(session, navigation) {
+async function resetStorageForUrl(session, url) {
   /** @type {Array<LH.IcuMessage>} */
   const warnings = [];
 
   // Reset the storage and warn if there appears to be other important data.
-  const warning = await storage.getImportantStorageWarning(session, navigation.url);
+  const warning = await storage.getImportantStorageWarning(session, url);
   if (warning) warnings.push(warning);
-  await storage.clearDataForOrigin(session, navigation.url);
+  await storage.clearDataForOrigin(session, url);
   await storage.clearBrowserCaches(session);
 
   return {warnings};
@@ -190,7 +190,7 @@ async function prepareTargetForNavigationMode(driver, settings) {
  *
  * @param {LH.Gatherer.FRProtocolSession} session
  * @param {LH.Config.Settings} settings
- * @param {Pick<LH.Config.NavigationDefn, 'disableThrottling'|'disableStorageReset'|'blockedUrlPatterns'> & {url: string}} navigation
+ * @param {Pick<LH.Config.NavigationDefn, 'disableThrottling'|'disableStorageReset'|'blockedUrlPatterns'> & {requestor: LH.NavigationRequestor}} navigation
  * @return {Promise<{warnings: Array<LH.IcuMessage>}>}
  */
 async function prepareTargetForIndividualNavigation(session, settings, navigation) {
@@ -200,9 +200,13 @@ async function prepareTargetForIndividualNavigation(session, settings, navigatio
   /** @type {Array<LH.IcuMessage>} */
   const warnings = [];
 
-  const shouldResetStorage = !settings.disableStorageReset && !navigation.disableStorageReset;
+  const {requestor} = navigation;
+  const shouldResetStorage =
+    !settings.disableStorageReset &&
+    !navigation.disableStorageReset &&
+    typeof requestor === 'string';
   if (shouldResetStorage) {
-    const {warnings: storageWarnings} = await resetStorageForNavigation(session, navigation);
+    const {warnings: storageWarnings} = await resetStorageForUrl(session, requestor);
     warnings.push(...storageWarnings);
   }
 

lighthouse-core/gather/gather-runner.js

@@ -572,7 +572,7 @@ class GatherRunner {
       driver.defaultSession,
       passContext.settings,
       {
-        url: passContext.url,
+        requestor: passContext.url,
         disableStorageReset: !passConfig.useThrottling,
         disableThrottling: !passConfig.useThrottling,
         blockedUrlPatterns: passConfig.blockedUrlPatterns,