core(user-flow): dry run flag

docs/user-flows.md

@@ -252,6 +252,7 @@ main();
 - Keep timespan recordings _short_ and focused on a single interaction sequence or page transition.
 - Use snapshot recordings when a substantial portion of the page content has changed.
 - Always wait for transitions and interactions to finish before ending a timespan. `page.waitForSelector`/`page.waitForFunction`/`page.waitForResponse`/`page.waitForTimeout` are your friends here.
+- Quickly validate your user flow by doing a dry run: `api.startFlow(page, {dryRun: true})`.
 
 ## Related Reading
 

lighthouse-core/fraggle-rock/gather/dry-run.js

@@ -0,0 +1,59 @@
+/**
+ * @license Copyright 2022 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const Driver = require('./driver.js');
+const emulation = require('../../lib/emulation.js');
+const {initializeConfig} = require('../config/config.js');
+const {gotoURL} = require('../../gather/driver/navigation.js');
+
+/**
+ * @param {LH.Gatherer.GatherMode} gatherMode
+ * @param {{page: LH.Puppeteer.Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
+ */
+async function dryRunSetup(gatherMode, options) {
+  const {page, config: configJson, configContext} = options;
+  const {config} = initializeConfig(configJson, {...configContext, gatherMode});
+  const driver = new Driver(page);
+  await driver.connect();
+  await emulation.emulate(driver.defaultSession, config.settings);
+  return {driver, config};
+}
+
+/**
+ * @param {Exclude<LH.Gatherer.GatherMode, 'navigation'>} gatherMode
+ * @param {{page: LH.Puppeteer.Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
+ */
+async function dryRun(gatherMode, options) {
+  const {driver} = await dryRunSetup(gatherMode, options);
+  await driver.disconnect();
+}
+
+/**
+ * @param {LH.NavigationRequestor} requestor
+ * @param {{page: LH.Puppeteer.Page, config?: LH.Config.Json, configContext?: LH.Config.FRContext}} options
+ */
+async function dryRunNavigation(requestor, options) {
+  const {driver, config} = await dryRunSetup('navigation', options);
+  if (!config.navigations || !config.navigations.length) {
+    throw new Error('No navigations configured');
+  }
+
+  const navigation = config.navigations[0];
+  await gotoURL(driver, requestor, {
+    ...navigation,
+    maxWaitForFcp: config.settings.maxWaitForFcp,
+    maxWaitForLoad: config.settings.maxWaitForLoad,
+    waitUntil: navigation.pauseAfterFcpMs ? ['fcp', 'load'] : ['load'],
+  });
+
+  await driver.disconnect();
+}
+
+module.exports = {
+  dryRun,
+  dryRunNavigation,
+};

lighthouse-core/fraggle-rock/user-flow.js

@@ -9,11 +9,12 @@ const {generateFlowReportHtml} = require('../../report/generator/report-generato
 const {snapshotGather} = require('./gather/snapshot-runner.js');
 const {startTimespanGather} = require('./gather/timespan-runner.js');
 const {navigationGather} = require('./gather/navigation-runner.js');
+const {dryRun, dryRunNavigation} = require('./gather/dry-run.js');
 const Runner = require('../runner.js');
 const {initializeConfig} = require('./config/config.js');
 
 /** @typedef {Parameters<snapshotGather>[0]} FrOptions */
-/** @typedef {Omit<FrOptions, 'page'> & {name?: string}} UserFlowOptions */
+/** @typedef {Omit<FrOptions, 'page'> & {name?: string, dryRun?: boolean}} UserFlowOptions */
 /** @typedef {Omit<FrOptions, 'page'> & {stepName?: string}} StepOptions */
 /** @typedef {WeakMap<LH.UserFlow.GatherStep, LH.Gatherer.FRGatherResult['runnerOptions']>} GatherStepRunnerOptions */
 
@@ -24,9 +25,11 @@ class UserFlow {
    */
   constructor(page, options) {
     /** @type {FrOptions} */
-    this.options = {page, ...options};
+    this._options = {page, ...options};
     /** @type {string|undefined} */
-    this.name = options?.name;
+    this._name = options?.name;
+    /** @type {boolean|undefined} */
+    this._dryRun = options?.dryRun;
     /** @type {LH.UserFlow.GatherStep[]} */
     this._gatherSteps = [];
     /** @type {GatherStepRunnerOptions} */
@@ -62,7 +65,7 @@ class UserFlow {
    * @param {StepOptions=} stepOptions
    */
   _getNextNavigationOptions(stepOptions) {
-    const options = {...this.options, ...stepOptions};
+    const options = {...this._options, ...stepOptions};
     const configContext = {...options.configContext};
     const settingsOverrides = {...configContext.settingsOverrides};
 
@@ -108,59 +111,68 @@ class UserFlow {
    */
   async navigate(requestor, stepOptions) {
     if (this.currentTimespan) throw new Error('Timespan already in progress');
-
     const options = this._getNextNavigationOptions(stepOptions);
-    const gatherResult = await navigationGather(requestor, options);
 
-    this._addGatherStep(gatherResult, options);
+    if (this._dryRun) {
+      await dryRunNavigation(requestor, options);
+      return;
+    }
 
-    return gatherResult;
+    const gatherResult = await navigationGather(requestor, options);
+    this._addGatherStep(gatherResult, options);
   }
 
   /**
    * @param {StepOptions=} stepOptions
    */
   async startTimespan(stepOptions) {
     if (this.currentTimespan) throw new Error('Timespan already in progress');
+    const options = {...this._options, ...stepOptions};
+
+    if (this._dryRun) {
+      await dryRun('timespan', options);
+      return;
+    }
 
-    const options = {...this.options, ...stepOptions};
     const timespan = await startTimespanGather(options);
     this.currentTimespan = {timespan, options};
   }
 
   async endTimespan() {
+    if (this._dryRun) return;
     if (!this.currentTimespan) throw new Error('No timespan in progress');
 
     const {timespan, options} = this.currentTimespan;
     const gatherResult = await timespan.endTimespanGather();
     this.currentTimespan = undefined;
 
     this._addGatherStep(gatherResult, options);
-
-    return gatherResult;
   }
 
   /**
    * @param {StepOptions=} stepOptions
    */
   async snapshot(stepOptions) {
     if (this.currentTimespan) throw new Error('Timespan already in progress');
+    const options = {...this._options, ...stepOptions};
 
-    const options = {...this.options, ...stepOptions};
-    const gatherResult = await snapshotGather(options);
+    if (this._dryRun) {
+      await dryRun('snapshot', options);
+      return;
+    }
 
+    const gatherResult = await snapshotGather(options);
     this._addGatherStep(gatherResult, options);
-
-    return gatherResult;
   }
 
   /**
    * @returns {Promise<LH.FlowResult>}
    */
   async createFlowResult() {
+    if (this._dryRun) throw new Error('Cannot get flow result from a dry run');
     return auditGatherSteps(this._gatherSteps, {
-      name: this.name,
-      config: this.options.config,
+      name: this._name,
+      config: this._options.config,
       gatherStepRunnerOptions: this._gatherStepRunnerOptions,
     });
   }
@@ -169,6 +181,7 @@ class UserFlow {
    * @return {Promise<string>}
    */
   async generateReport() {
+    if (this._dryRun) throw new Error('Cannot generate a flow report from a dry run');
     const flowResult = await this.createFlowResult();
     return generateFlowReportHtml(flowResult);
   }
@@ -177,9 +190,10 @@ class UserFlow {
    * @return {LH.UserFlow.FlowArtifacts}
    */
   createArtifactsJson() {
+    if (this._dryRun) throw new Error('Cannot create flow artifacts from a dry run');
     return {
       gatherSteps: this._gatherSteps,
-      name: this.name,
+      name: this._name,
     };
   }
 }

lighthouse-core/scripts/update-flow-fixtures.js

@@ -88,7 +88,7 @@ async function rebaselineArtifacts(artifactKeys) {
   });
 
   const page = await browser.newPage();
-  const flow = await api.startFlow(page, {config});
+  const flow = await api.startFlow(page, {config, dryRun: true});
 
   await flow.navigate('https://www.mikescerealshack.co');
 

lighthouse-core/test/fraggle-rock/scenarios/dry-run-test-pptr.js

@@ -0,0 +1,58 @@
+/**
+ * @license Copyright 2022 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+import {jest} from '@jest/globals';
+
+import * as api from '../../../fraggle-rock/api.js';
+import {createTestState} from './pptr-test-utils.js';
+import {LH_ROOT} from '../../../../root.js';
+
+/* eslint-env jest */
+/* eslint-env browser */
+
+jest.setTimeout(90_000);
+
+describe('Dry Run', () => {
+  const state = createTestState();
+
+  state.installSetupAndTeardownHooks();
+
+  beforeAll(() => {
+    state.server.baseDir = `${LH_ROOT}/lighthouse-core/test/fixtures/fraggle-rock/snapshot-basic`;
+  });
+
+  it('should setup environment and perform flow actions', async () => {
+    const pageUrl = `${state.serverBaseUrl}/onclick.html`;
+
+    const flow = await api.startFlow(state.page, {dryRun: true});
+    await flow.navigate(pageUrl);
+
+    await flow.startTimespan();
+    await state.page.click('button');
+    await flow.endTimespan();
+
+    await flow.snapshot();
+
+    // Ensure page navigated occurred and button was clicked.
+    const finalUrl = await state.page.url();
+    expect(finalUrl).toEqual(`${pageUrl}#done`);
+
+    // Ensure Lighthouse emulated a mobile device.
+    const deviceMetrics = await state.page.evaluate(() => ({
+      width: document.documentElement.clientWidth,
+      height: document.documentElement.clientHeight,
+    }));
+    expect(deviceMetrics).toEqual({
+      height: 640,
+      width: 360,
+    });
+
+    expect(() => flow.createArtifactsJson()).toThrow();
+    await expect(flow.createFlowResult()).rejects.toThrow();
+    await expect(flow.generateReport()).rejects.toThrow();
+  });
+});

lighthouse-core/test/fraggle-rock/scenarios/pptr-test-utils.js

@@ -40,6 +40,7 @@ export function createTestState() {
         this.serverBaseUrl = `http://localhost:${this.server.getPort()}`;
         this.browser = await puppeteer.launch({
           headless: true,
+          ignoreDefaultArgs: ['--enable-automation'],
           executablePath: process.env.CHROME_PATH,
         });
       });

lighthouse-core/test/fraggle-rock/user-flow-test.js

@@ -15,6 +15,8 @@ const navigationModule = {navigationGather: jest.fn()};
 jest.mock('../../fraggle-rock/gather/navigation-runner.js', () => navigationModule);
 const timespanModule = {startTimespanGather: jest.fn()};
 jest.mock('../../fraggle-rock/gather/timespan-runner.js', () => timespanModule);
+const dryRunModule = {dryRun: jest.fn(), dryRunNavigation: jest.fn()};
+jest.mock('../../fraggle-rock/gather/dry-run.js', () => dryRunModule);
 
 const mockRunner = mockRunnerModule();
 
@@ -28,6 +30,9 @@ describe('UserFlow', () => {
 
     mockRunner.reset();
 
+    dryRunModule.dryRun.mockReset();
+    dryRunModule.dryRunNavigation.mockReset();
+
     snapshotModule.snapshotGather.mockReset();
     snapshotModule.snapshotGather.mockResolvedValue({
       artifacts: {
@@ -150,6 +155,14 @@ describe('UserFlow', () => {
       expect(configContext).toEqual({settingsOverrides: {maxWaitForLoad: 1000}});
       expect(configContextExplicit).toEqual({skipAboutBlank: false});
     });
+
+    it('should perform a minimal navigation in a dry run', async () => {
+      const flow = new UserFlow(mockPage.asPage(), {dryRun: true});
+      await flow.navigate('https://example.com/1');
+
+      expect(navigationModule.navigationGather).not.toHaveBeenCalled();
+      expect(dryRunModule.dryRunNavigation).toHaveBeenCalled();
+    });
   });
 
   describe('.startTimespan()', () => {
@@ -174,13 +187,29 @@ describe('UserFlow', () => {
         {name: 'Timespan report (www.example.com/)'},
       ]);
     });
+
+    it('should setup emulation in a dry run', async () => {
+      const flow = new UserFlow(mockPage.asPage(), {dryRun: true});
+      await flow.startTimespan();
+
+      expect(timespanModule.startTimespanGather).not.toHaveBeenCalled();
+      expect(dryRunModule.dryRun).toHaveBeenCalled();
+    });
   });
 
   describe('.endTimespan()', () => {
     it('should throw if a timespan has not started', async () => {
       const flow = new UserFlow(mockPage.asPage());
       await expect(flow.endTimespan()).rejects.toBeTruthy();
     });
+
+    it('should do nothing in a dry run', async () => {
+      const flow = new UserFlow(mockPage.asPage(), {dryRun: true});
+      await flow.endTimespan();
+
+      expect(timespanModule.startTimespanGather).not.toHaveBeenCalled();
+      expect(dryRunModule.dryRun).not.toHaveBeenCalled();
+    });
   });
 
   describe('.snapshot()', () => {
@@ -202,6 +231,14 @@ describe('UserFlow', () => {
         {name: 'Snapshot report (www.example.com/)'},
       ]);
     });
+
+    it('should setup emulation in a dry run', async () => {
+      const flow = new UserFlow(mockPage.asPage(), {dryRun: true});
+      await flow.snapshot();
+
+      expect(snapshotModule.snapshotGather).not.toHaveBeenCalled();
+      expect(dryRunModule.dryRun).toHaveBeenCalled();
+    });
   });
 
   describe('.getFlowResult', () => {
@@ -211,6 +248,13 @@ describe('UserFlow', () => {
       await expect(flowResultPromise).rejects.toThrow(/Need at least one step/);
     });
 
+    it('should throw after a dry run', async () => {
+      const flow = new UserFlow(mockPage.asPage(), {dryRun: true});
+      await flow.snapshot();
+      const flowResultPromise = flow.createFlowResult();
+      await expect(flowResultPromise).rejects.toThrow(/Cannot.*dry run/);
+    });
+
     it('should audit active gather steps', async () => {
       mockRunner.audit.mockImplementation(artifacts => ({
         lhr: {