diff --git a/lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js b/lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js index 19b5dcc25446..8662f73059e5 100644 --- a/lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js +++ b/lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js @@ -186,6 +186,7 @@ async function begin() { console.log('\n✨ Be sure to have recently run this: yarn build-all'); } const {runLighthouse} = await import(runnerPath); + runLighthouse.runnerName = argv.runner; // Find test definition file and filter by requestedTestIds. let testDefnPath = argv.testsPath || coreTestDefnsPath; diff --git a/lighthouse-cli/test/smokehouse/readme.md b/lighthouse-cli/test/smokehouse/readme.md index 0a5875fe5c57..ea3283277158 100644 --- a/lighthouse-cli/test/smokehouse/readme.md +++ b/lighthouse-cli/test/smokehouse/readme.md @@ -66,6 +66,8 @@ However, if an array literal is used as the expectation, an extra condition is e Arrays can be checked against a subset of elements using the special `_includes` property. The value of `_includes` _must_ be an array. Each assertion in `_includes` will remove the matching item from consideration for the rest. +Arrays can be asserted to not match any elements using the special `_excludes` property. The value of `_excludes` _must_ be an array. If an `_includes` check is defined before an `_excludes` check, only the element not matched under the previous will be considered. + **Examples**: | Actual | Expected | Result | | -- | -- | -- | @@ -73,6 +75,9 @@ Arrays can be checked against a subset of elements using the special `_includes` | `[{timeInMs: 5}, {timeInMs: 15}]` | `{length: 2}` | ✅ PASS | | `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}]}` | ✅ PASS | | `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}, {timeInMs: 5}]}` | ❌ FAIL | +| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}], _excludes: [{timeInMs: 5}]}` | ✅ PASS | +| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}], _excludes: [{timeInMs: 15}]}` | ❌ FAIL | +| `[{timeInMs: 5}, {timeInMs: 15}]` | `{_includes: [{timeInMs: 5}], _excludes: [{}]}` | ❌ FAIL | | `[{timeInMs: 5}, {timeInMs: 15}]` | `[{timeInMs: 5}]` | ❌ FAIL | ### Special environment checks @@ -104,6 +109,15 @@ If an expectation requires a minimum version of Chromium, use `_minChromiumMiles }, ``` +All pruning checks: + +- `_minChromiumMilestone` +- `_maxChromiumMilestone` +- `_legacyOnly` +- `_fraggleRockOnly` +- `_skipInBundled` +- `_runner` (set to same value provided to CLI --runner flag, ex: `'devtools'`) + ## Pipeline The different frontends launch smokehouse with a set of tests to run. Smokehouse then coordinates the tests using a particular method of running Lighthouse (CLI, as a bundle, etc). diff --git a/lighthouse-cli/test/smokehouse/report-assert.js b/lighthouse-cli/test/smokehouse/report-assert.js index 88efaf129585..5f33c48da7af 100644 --- a/lighthouse-cli/test/smokehouse/report-assert.js +++ b/lighthouse-cli/test/smokehouse/report-assert.js @@ -103,6 +103,8 @@ function findDifference(path, actual, expected) { }; } + let inclExclCopy; + // We only care that all expected's own properties are on actual (and not the other way around). // Note an expected `undefined` can match an actual that is either `undefined` or not defined. for (const key of Object.keys(expected)) { @@ -112,6 +114,8 @@ function findDifference(path, actual, expected) { const expectedValue = expected[key]; if (key === '_includes') { + inclExclCopy = [...actual]; + if (!Array.isArray(expectedValue)) throw new Error('Array subset must be array'); if (!Array.isArray(actual)) { return { @@ -121,12 +125,12 @@ function findDifference(path, actual, expected) { }; } - const actualCopy = [...actual]; for (const expectedEntry of expectedValue) { const matchingIndex = - actualCopy.findIndex(actualEntry => !findDifference(keyPath, actualEntry, expectedEntry)); + inclExclCopy.findIndex(actualEntry => + !findDifference(keyPath, actualEntry, expectedEntry)); if (matchingIndex !== -1) { - actualCopy.splice(matchingIndex, 1); + inclExclCopy.splice(matchingIndex, 1); continue; } @@ -140,6 +144,33 @@ function findDifference(path, actual, expected) { continue; } + if (key === '_excludes') { + // Re-use state from `_includes` check, if there was one. + /** @type {any[]} */ + const arrToCheckAgainst = inclExclCopy || actual; + + if (!Array.isArray(expectedValue)) throw new Error('Array subset must be array'); + if (!Array.isArray(actual)) continue; + + const expectedExclusions = expectedValue; + for (const expectedExclusion of expectedExclusions) { + const matchingIndex = arrToCheckAgainst.findIndex(actualEntry => + !findDifference(keyPath, actualEntry, expectedExclusion)); + if (matchingIndex !== -1) { + return { + path, + actual: arrToCheckAgainst[matchingIndex], + expected: { + message: 'Expected to not find matching entry via _excludes', + expectedExclusion, + }, + }; + } + } + + continue; + } + const actualValue = actual[key]; const subDifference = findDifference(keyPath, actualValue, expectedValue); @@ -187,7 +218,7 @@ function makeComparison(name, actualResult, expectedResult) { * @param {LocalConsole} localConsole * @param {LH.Result} lhr * @param {Smokehouse.ExpectedRunnerResult} expected - * @param {{isBundled?: boolean}=} reportOptions + * @param {{runner?: string, isBundled?: boolean}=} reportOptions */ function pruneExpectations(localConsole, lhr, expected, reportOptions) { const isFraggleRock = lhr.configSettings.channel === 'fraggle-rock-cli'; @@ -217,8 +248,20 @@ function pruneExpectations(localConsole, lhr, expected, reportOptions) { * @param {*} obj */ function pruneRecursively(obj) { - for (const key of Object.keys(obj)) { - const value = obj[key]; + /** + * @param {string} key + */ + const remove = (key) => { + if (Array.isArray(obj)) { + obj.splice(Number(key), 1); + } else { + delete obj[key]; + } + }; + + // Because we may be deleting keys, we should iterate the keys backwards + // otherwise arrays with multiple pruning checks will skip elements. + for (const [key, value] of Object.entries(obj).reverse()) { if (!value || typeof value !== 'object') { continue; } @@ -229,42 +272,32 @@ function pruneExpectations(localConsole, lhr, expected, reportOptions) { JSON.stringify(value, null, 2), `Actual Chromium version: ${getChromeVersion()}`, ].join(' ')); - if (Array.isArray(obj)) { - obj.splice(Number(key), 1); - } else { - delete obj[key]; - } + remove(key); } else if (value._legacyOnly && isFraggleRock) { localConsole.log([ `[${key}] marked legacy only but run is Fraggle Rock, pruning expectation:`, JSON.stringify(value, null, 2), ].join(' ')); - if (Array.isArray(obj)) { - obj.splice(Number(key), 1); - } else { - delete obj[key]; - } + remove(key); } else if (value._fraggleRockOnly && !isFraggleRock) { localConsole.log([ `[${key}] marked Fraggle Rock only but run is legacy, pruning expectation:`, JSON.stringify(value, null, 2), `Actual channel: ${lhr.configSettings.channel}`, ].join(' ')); - if (Array.isArray(obj)) { - obj.splice(Number(key), 1); - } else { - delete obj[key]; - } + remove(key); } else if (value._skipInBundled && !isBundled) { localConsole.log([ `[${key}] marked as skip in bundled and runner is bundled, pruning expectation:`, JSON.stringify(value, null, 2), ].join(' ')); - if (Array.isArray(obj)) { - obj.splice(Number(key), 1); - } else { - delete obj[key]; - } + remove(key); + } else if (value._runner && reportOptions?.runner !== value._runner) { + localConsole.log([ + `[${key}] is only for runner ${value._runner}, pruning expectation:`, + JSON.stringify(value, null, 2), + ].join(' ')); + remove(key); } else { pruneRecursively(value); } @@ -275,6 +308,7 @@ function pruneExpectations(localConsole, lhr, expected, reportOptions) { delete obj._skipInBundled; delete obj._minChromiumMilestone; delete obj._maxChromiumMilestone; + delete obj._runner; } const cloned = cloneDeep(expected); @@ -420,7 +454,7 @@ function reportAssertion(localConsole, assertion) { * summary. Returns count of passed and failed tests. * @param {{lhr: LH.Result, artifacts: LH.Artifacts, networkRequests?: string[]}} actual * @param {Smokehouse.ExpectedRunnerResult} expected - * @param {{isDebug?: boolean, isBundled?: boolean}=} reportOptions + * @param {{runner?: string, isDebug?: boolean, isBundled?: boolean}=} reportOptions * @return {{passed: number, failed: number, log: string}} */ function getAssertionReport(actual, expected, reportOptions = {}) { diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js index 139dd6b0f7bd..5177119b9e12 100644 --- a/lighthouse-cli/test/smokehouse/smokehouse.js +++ b/lighthouse-cli/test/smokehouse/smokehouse.js @@ -57,7 +57,7 @@ async function runSmokehouse(smokeTestDefns, smokehouseOptions) { useFraggleRock, jobs = DEFAULT_CONCURRENT_RUNS, retries = DEFAULT_RETRIES, - lighthouseRunner = cliLighthouseRunner, + lighthouseRunner = Object.assign(cliLighthouseRunner, {runnerName: 'cli'}), takeNetworkRequestUrls, } = smokehouseOptions; assertPositiveInteger('jobs', jobs); @@ -159,7 +159,10 @@ async function runSmokeTest(smokeTestDefn, testOptions) { } // Assert result. - report = getAssertionReport(result, expectations, {isDebug}); + report = getAssertionReport(result, expectations, { + runner: lighthouseRunner.runnerName, + isDebug, + }); runs.push({ ...result, diff --git a/types/smokehouse.d.ts b/types/smokehouse.d.ts index b2bcc364a08f..a60c5e3aa5a2 100644 --- a/types/smokehouse.d.ts +++ b/types/smokehouse.d.ts @@ -52,7 +52,7 @@ declare global { {expectations: Smokehouse.ExpectedRunnerResult | Array} export type LighthouseRunner = - (url: string, configJson?: Config.Json, runnerOptions?: {isDebug?: boolean; useFraggleRock?: boolean}) => Promise<{lhr: LHResult, artifacts: Artifacts, log: string}>; + {runnerName?: string} & ((url: string, configJson?: Config.Json, runnerOptions?: {isDebug?: boolean; useFraggleRock?: boolean}) => Promise<{lhr: LHResult, artifacts: Artifacts, log: string}>); export interface SmokehouseOptions { /** If true, performs extra logging from the test runs. */