tests: use --headless=new for all smoketests (#14419)

.github/workflows/devtools.yml

@@ -168,7 +168,7 @@ jobs:
     - run: mkdir latest-run
       working-directory: ${{ github.workspace }}/lighthouse
     - name: yarn smoke --runner devtools
-      run: xvfb-run --auto-servernum yarn smoke --runner devtools --shard=${{ matrix.smoke-test-shard }}/${{ strategy.job-total }} --jobs=3 --retries=2 --debug
+      run: yarn smoke --runner devtools --shard=${{ matrix.smoke-test-shard }}/${{ strategy.job-total }} --jobs=3 --retries=2 --debug
       working-directory: ${{ github.workspace }}/lighthouse
 
     - name: Upload failures

.github/workflows/smoke.yml

@@ -54,10 +54,9 @@ jobs:
     - run: yarn build-report
     - run: yarn reset-link
 
-    - run: sudo apt-get install xvfb
     - name: Run smoke tests
       run: |
-        xvfb-run --auto-servernum yarn c8 yarn smoke --debug -j=2 --retries=2 --shard=${{ matrix.smoke-test-shard }}/$SHARD_TOTAL
+        yarn c8 yarn smoke --debug -j=2 --retries=2 --shard=${{ matrix.smoke-test-shard }}/$SHARD_TOTAL
         yarn c8 report --reporter text-lcov > smoke-coverage.lcov
 
     - name: Upload test coverage to Codecov
@@ -154,9 +153,8 @@ jobs:
       working-directory: /home/runner
       run: bash $GITHUB_WORKSPACE/core/scripts/download-chrome.sh && mv chrome-linux chrome-linux-tot
 
-    - run: sudo apt-get install xvfb
     - name: yarn test-bundle
-      run: xvfb-run --auto-servernum yarn test-bundle --shard=${{ matrix.smoke-test-shard }}/$SHARD_TOTAL --retries=2
+      run: yarn test-bundle --shard=${{ matrix.smoke-test-shard }}/$SHARD_TOTAL --retries=2
 
     # Fail if any changes were written to source files.
     - run: git diff --exit-code

cli/test/smokehouse/frontends/smokehouse-bin.js

@@ -21,7 +21,7 @@ import yargs from 'yargs';
 import * as yargsHelpers from 'yargs/helpers';
 import log from 'lighthouse-logger';
 
-import {runSmokehouse, getShardedDefinitions} from '../smokehouse.js';
+import {runSmokehouse, getShardedDefinitions, DEFAULT_RETRIES, DEFAULT_CONCURRENT_RUNS} from '../smokehouse.js';
 import {updateTestDefnFormat} from './back-compat-util.js';
 import {LH_ROOT} from '../../../../shared/root.js';
 import exclusions from '../config/exclusions.js';
@@ -139,10 +139,12 @@ async function begin() {
       'jobs': {
         type: 'number',
         alias: 'j',
+        default: DEFAULT_CONCURRENT_RUNS,
         describe: 'Manually set the number of jobs to run at once. `1` runs all tests serially',
       },
       'retries': {
         type: 'number',
+        default: DEFAULT_RETRIES,
         describe: 'The number of times to retry failing tests before accepting. Defaults to 0',
       },
       'runner': {
@@ -164,6 +166,15 @@ async function begin() {
         default: false,
         describe: 'Ignore any smoke test exclusions set.',
       },
+      'headless': {
+        type: 'boolean',
+        default: true,
+        hidden: true,
+      },
+      'no-headless': {
+        type: 'boolean',
+        describe: 'Launch Chrome in typical desktop headful mode, rather than our default of `--headless=new` (https://developer.chrome.com/articles/new-headless/).', // eslint-disable-line max-len
+      },
     })
     .wrap(y.terminalWidth())
     .argv;
@@ -173,9 +184,6 @@ async function begin() {
   const argv =
     /** @type {Awaited<typeof rawArgv> & LH.Util.CamelCasify<Awaited<typeof rawArgv>>} */ (rawArgv);
 
-  const jobs = Number.isFinite(argv.jobs) ? argv.jobs : undefined;
-  const retries = Number.isFinite(argv.retries) ? argv.retries : undefined;
-
   const runnerPath = runnerPaths[/** @type {keyof typeof runnerPaths} */ (argv.runner)];
   if (argv.runner === 'bundle') {
     console.log('\n✨ Be sure to have recently run this: yarn build-all');
@@ -210,9 +218,12 @@ async function begin() {
 
     const prunedTestDefns = pruneExpectedNetworkRequests(testDefns, takeNetworkRequestUrls);
     const options = {
-      jobs,
-      retries,
-      isDebug: argv.debug,
+      jobs: argv.jobs,
+      retries: argv.retries,
+      testRunnerOptions: {
+        isDebug: argv.debug,
+        headless: argv.headless,
+      },
       lighthouseRunner: runLighthouse,
       takeNetworkRequestUrls,
       setup,

cli/test/smokehouse/lighthouse-runners/bundle.js

@@ -46,7 +46,7 @@ if (!isMainThread && parentPort) {
 /**
  * @param {string} url
  * @param {LH.Config|undefined} config
- * @param {{isDebug?: boolean}} testRunnerOptions
+ * @param {Smokehouse.SmokehouseOptions['testRunnerOptions']} testRunnerOptions
  * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts}>}
  */
 async function runBundledLighthouse(url, config, testRunnerOptions) {
@@ -74,12 +74,16 @@ async function runBundledLighthouse(url, config, testRunnerOptions) {
   const lighthouse = global.runBundledLighthouse;
 
   // Launch and connect to Chrome.
-  const launchedChrome = await ChromeLauncher.launch();
+  const launchedChrome = await ChromeLauncher.launch({
+    chromeFlags: [
+      testRunnerOptions?.headless ? '--headless=new' : '',
+    ],
+  });
   const port = launchedChrome.port;
 
   // Run Lighthouse.
   try {
-    const logLevel = testRunnerOptions.isDebug ? 'verbose' : 'info';
+    const logLevel = testRunnerOptions?.isDebug ? 'verbose' : 'info';
 
     // Puppeteer is not included in the bundle, we must create the page here.
     const browser = await puppeteer.connect({browserURL: `http://127.0.0.1:${port}`});
@@ -101,7 +105,7 @@ async function runBundledLighthouse(url, config, testRunnerOptions) {
  * Launch Chrome and do a full Lighthouse run via the Lighthouse DevTools bundle.
  * @param {string} url
  * @param {LH.Config=} config
- * @param {{isDebug?: boolean}=} testRunnerOptions
+ * @param {Smokehouse.SmokehouseOptions['testRunnerOptions']=} testRunnerOptions
  * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
  */
 async function runLighthouse(url, config, testRunnerOptions = {}) {

cli/test/smokehouse/lighthouse-runners/cli.js

@@ -28,7 +28,7 @@ const execFileAsync = promisify(execFile);
  * Launch Chrome and do a full Lighthouse run via the Lighthouse CLI.
  * @param {string} url
  * @param {LH.Config=} config
- * @param {{isDebug?: boolean}=} testRunnerOptions
+ * @param {Smokehouse.SmokehouseOptions['testRunnerOptions']=} testRunnerOptions
  * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
  */
 async function runLighthouse(url, config, testRunnerOptions = {}) {
@@ -46,11 +46,11 @@ async function runLighthouse(url, config, testRunnerOptions = {}) {
  * @param {string} url
  * @param {string} tmpPath
  * @param {LH.Config=} config
- * @param {{isDebug?: boolean}=} options
+ * @param {Smokehouse.SmokehouseOptions['testRunnerOptions']=} options
  * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
  */
 async function internalRun(url, tmpPath, config, options) {
-  const {isDebug = false} = options || {};
+  const {isDebug, headless} = options || {};
   const localConsole = new LocalConsole();
 
   const outputPath = `${tmpPath}/smokehouse.report.json`;
@@ -67,6 +67,8 @@ async function internalRun(url, tmpPath, config, options) {
     '--quiet',
   ];
 
+  if (headless) args.push('--chrome-flags="--headless=new"');
+
   // Config can be optionally provided.
   if (config) {
     const configPath = `${tmpPath}/config.json`;

cli/test/smokehouse/lighthouse-runners/devtools.js

@@ -40,10 +40,12 @@ async function setup() {
  * CHROME_PATH determines which Chrome is used–otherwise the default is puppeteer's chrome binary.
  * @param {string} url
  * @param {LH.Config=} config
+ * @param {Smokehouse.SmokehouseOptions['testRunnerOptions']=} testRunnerOptions
  * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>}
  */
-async function runLighthouse(url, config) {
+async function runLighthouse(url, config, testRunnerOptions) {
   const chromeFlags = [
+    testRunnerOptions?.headless ? '--headless=new' : '',
     `--custom-devtools-frontend=file://${devtoolsDir}/out/LighthouseIntegration/gen/front_end`,
   ];
   const {lhr, artifacts, logs} = await testUrlFromDevtools(url, {

cli/test/smokehouse/smokehouse.js

@@ -46,12 +46,12 @@ const DEFAULT_RETRIES = 0;
 /**
  * Runs the selected smoke tests. Returns whether all assertions pass.
  * @param {Array<Smokehouse.TestDfn>} smokeTestDefns
- * @param {Smokehouse.SmokehouseOptions} smokehouseOptions
+ * @param {Partial<Smokehouse.SmokehouseOptions>} smokehouseOptions
  * @return {Promise<{success: boolean, testResults: SmokehouseResult[]}>}
  */
 async function runSmokehouse(smokeTestDefns, smokehouseOptions) {
   const {
-    isDebug,
+    testRunnerOptions,
     jobs = DEFAULT_CONCURRENT_RUNS,
     retries = DEFAULT_RETRIES,
     lighthouseRunner = Object.assign(cliLighthouseRunner, {runnerName: 'cli'}),
@@ -73,7 +73,8 @@ async function runSmokehouse(smokeTestDefns, smokehouseOptions) {
   const concurrentMapper = new ConcurrentMapper();
 
   const testOptions = {
-    isDebug,
+    testRunnerOptions,
+    jobs,
     retries,
     lighthouseRunner,
     takeNetworkRequestUrls,
@@ -133,15 +134,15 @@ function purpleify(str) {
 /**
  * Run Lighthouse in the selected runner.
  * @param {Smokehouse.TestDfn} smokeTestDefn
- * @param {{isDebug?: boolean, retries: number, lighthouseRunner: Smokehouse.LighthouseRunner, takeNetworkRequestUrls?: () => string[]}} testOptions
+ * @param {Smokehouse.SmokehouseOptions} testOptions
  * @return {Promise<SmokehouseResult>}
  */
 async function runSmokeTest(smokeTestDefn, testOptions) {
   const {id, expectations, config} = smokeTestDefn;
   const {
     lighthouseRunner,
     retries,
-    isDebug,
+    testRunnerOptions,
     takeNetworkRequestUrls,
   } = testOptions;
   const requestedUrl = expectations.lhr.requestedUrl;
@@ -163,7 +164,7 @@ async function runSmokeTest(smokeTestDefn, testOptions) {
     // Run Lighthouse.
     try {
       result = {
-        ...await lighthouseRunner(requestedUrl, config, {isDebug}),
+        ...await lighthouseRunner(requestedUrl, config, testRunnerOptions),
         networkRequests: takeNetworkRequestUrls ? takeNetworkRequestUrls() : undefined,
       };
 
@@ -183,7 +184,7 @@ async function runSmokeTest(smokeTestDefn, testOptions) {
     // Assert result.
     report = getAssertionReport(result, expectations, {
       runner: lighthouseRunner.runnerName,
-      isDebug,
+      ...testRunnerOptions,
     });
 
     runs.push({
@@ -310,4 +311,6 @@ function getShardedDefinitions(testDefns, shardArg) {
 export {
   runSmokehouse,
   getShardedDefinitions,
+  DEFAULT_RETRIES,
+  DEFAULT_CONCURRENT_RUNS,
 };

docs/headless-chrome.md

@@ -22,6 +22,11 @@ Kick off run of Lighthouse using headless Chrome:
 lighthouse --chrome-flags="--headless" https://github.com
 ```
 
+## (CLI headless=new)
+
+There is also the new `--headless=new` option, which includes functionality that
+was explicitly omitted from the original headless browser.
+
 ## CLI (xvfb)
 
 Alternatively, you can run full Chrome + xvfb instead of headless mode. These steps worked on Debian Jessie:

types/internal/smokehouse.d.ts

@@ -56,14 +56,19 @@ declare global {
       {runnerName?: string} & ((url: string, config?: Config, runnerOptions?: {isDebug?: boolean}) => Promise<{lhr: LHResult, artifacts: Artifacts, log: string}>);
 
     export interface SmokehouseOptions {
-      /** If true, performs extra logging from the test runs. */
-      isDebug?: boolean;
+      /** Options to pass to the specific Lighthouse runner. */
+      testRunnerOptions?: {
+        /** If true, performs extra logging from the test runs. */
+        isDebug?: boolean;
+        /** Launch Chrome in the new headless mode (`--headless=new`), rather than the typical desktop headful mode. */
+        headless?: boolean;
+      };
       /** Manually set the number of jobs to run at once. `1` runs all tests serially. */
-      jobs?: number;
+      jobs: number;
       /** The number of times to retry failing tests before accepting. Defaults to 0. */
-      retries?: number;
+      retries: number;
       /** A function that runs Lighthouse with the given options. Defaults to running Lighthouse via the CLI. */
-      lighthouseRunner?: LighthouseRunner;
+      lighthouseRunner: LighthouseRunner;
       /** A function that gets a list of URLs requested to the server since the last fetch. */
       takeNetworkRequestUrls?: () => string[];
       /** A function run once before all smoke tests. */