From 191bb5824f5f4e5ee41c2aaf6079ace788c35e3a Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Mon, 13 Feb 2023 16:25:25 -0500 Subject: [PATCH 1/5] add basic support for testing matrix --- __tests__/core/runner.test.ts | 2 +- src/common_types.ts | 7 +++ src/core/index.ts | 34 +++++++++++-- src/core/runner.ts | 18 ++++--- src/dsl/journey.ts | 8 ++- src/matrix.ts | 96 +++++++++++++++++++++++++++++++++++ src/options.ts | 5 ++ 7 files changed, 156 insertions(+), 14 deletions(-) create mode 100644 src/matrix.ts diff --git a/__tests__/core/runner.test.ts b/__tests__/core/runner.test.ts index c069e5a5..f2287675 100644 --- a/__tests__/core/runner.test.ts +++ b/__tests__/core/runner.test.ts @@ -729,7 +729,7 @@ describe('runner', () => { type: 'browser', tags: [], locations: ['united_kingdom'], - privaateLocations: undefined, + privateLocations: undefined, schedule: 3, params: undefined, playwrightOptions: undefined, diff --git a/src/common_types.ts b/src/common_types.ts index ce3ddd2f..2b6413f4 100644 --- a/src/common_types.ts +++ b/src/common_types.ts @@ -237,6 +237,7 @@ export type RunOptions = BaseArgs & { environment?: string; playwrightOptions?: PlaywrightOptions; networkConditions?: NetworkConditions; + matrix?: Matrix; reporter?: BuiltInReporterName | ReporterInstance; }; @@ -254,12 +255,18 @@ export type ProjectSettings = { space: string; }; +export type Matrix = { + values: Record; + adjustments?: Array>; +} + export type PlaywrightOptions = LaunchOptions & BrowserContextOptions; export type SyntheticsConfig = { params?: Params; playwrightOptions?: PlaywrightOptions; monitor?: MonitorConfig; project?: ProjectSettings; + matrix?: Matrix; }; /** Runner Payload types */ diff --git a/src/core/index.ts b/src/core/index.ts index ce806e6a..ac4fa7ce 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -22,14 +22,21 @@ * THE SOFTWARE. * */ - import { Journey, JourneyCallback, JourneyOptions } from '../dsl'; import Runner from './runner'; import { VoidCallback, HooksCallback, Location } from '../common_types'; +import { normalizeOptions } from '../options'; import { wrapFnWithLocation } from '../helpers'; +import { getCombinations, getCombinationName } from '../matrix'; import { log } from './logger'; import { MonitorConfig } from '../dsl/monitor'; + +/* TODO: Testing + * Local vs global matrix: Local matrix fully overwrites global matrix, rather than merging + * Adjustments: Duplicates in adjustments do not run extra journeys + * + /** * Use a gloabl Runner which would be accessed by the runtime and * required to handle the local vs global invocation through CLI @@ -38,7 +45,6 @@ const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER'); if (!global[SYNTHETICS_RUNNER]) { global[SYNTHETICS_RUNNER] = new Runner(); } - export const runner: Runner = global[SYNTHETICS_RUNNER]; export const journey = wrapFnWithLocation( @@ -51,9 +57,27 @@ export const journey = wrapFnWithLocation( if (typeof options === 'string') { options = { name: options, id: options }; } - const j = new Journey(options, callback, location); - runner.addJourney(j); - return j; + const { matrix: globalMatrix } = normalizeOptions({}); + const { matrix: journeyMatrix } = options; + if (!globalMatrix && !journeyMatrix) { + const j = new Journey(options, callback, location); + runner.addJourney(j); + return j; + } + + // local journey matrix takes priority over global matrix + const matrix = journeyMatrix || globalMatrix; + + if (!matrix.values) { + throw new Error('Please specify values for your testing matrix'); + } + + const combinations = getCombinations(matrix); + combinations.forEach(matrixParams => { + const name = getCombinationName((options as JourneyOptions)?.name, matrixParams); + const j = new Journey({...options as JourneyOptions, name}, callback, location, matrixParams); + runner.addJourney(j); + }) } ); diff --git a/src/core/runner.ts b/src/core/runner.ts index d3dee99e..30dba2e3 100644 --- a/src/core/runner.ts +++ b/src/core/runner.ts @@ -339,16 +339,18 @@ export default class Runner { async runJourney(journey: Journey, options: RunOptions) { const result: JourneyResult = { status: 'succeeded' }; - const context = await Runner.createContext(options); + const params = Object.freeze({...options.params, ...journey.params}); + const journeyOptions = { ...options, params}; + const context = await Runner.createContext(journeyOptions); log(`Runner: start journey (${journey.name})`); try { this.registerJourney(journey, context); const hookArgs = { env: options.environment, - params: options.params, + params: params, }; await this.runBeforeHook(journey, hookArgs); - const stepResults = await this.runSteps(journey, context, options); + const stepResults = await this.runSteps(journey, context, journeyOptions); /** * Mark journey as failed if any intermediate step fails */ @@ -363,7 +365,7 @@ export default class Runner { result.status = 'failed'; result.error = e; } finally { - await this.endJourney(journey, { ...context, ...result }, options); + await this.endJourney(journey, { ...context, ...result }, journeyOptions); await Gatherer.dispose(context.driver); } log(`Runner: end journey (${journey.name})`); @@ -399,6 +401,7 @@ export default class Runner { const { match, tags } = options; const monitors: Monitor[] = []; for (const journey of this.journeys) { + const params = Object.freeze({ ...this.monitor.config?.params, ...options.params, ...journey.params }); if (!journey.isMatch(match, tags)) { continue; } @@ -407,8 +410,11 @@ export default class Runner { * Execute dummy callback to get all monitor specific * configurations for the current journey */ - journey.callback({ params: options.params } as any); - journey.monitor.update(this.monitor?.config); + journey.callback({ params: params } as any); + journey.monitor.update({ + ...this.monitor?.config, + params: Object.keys(params).length ? params : undefined + }); journey.monitor.validate(); monitors.push(journey.monitor); } diff --git a/src/dsl/journey.ts b/src/dsl/journey.ts index b9bb7200..29c4dff0 100644 --- a/src/dsl/journey.ts +++ b/src/dsl/journey.ts @@ -32,13 +32,14 @@ import { } from 'playwright-chromium'; import micromatch, { isMatch } from 'micromatch'; import { Step } from './step'; -import { VoidCallback, HooksCallback, Params, Location } from '../common_types'; +import { VoidCallback, HooksCallback, Params, Location, Matrix } from '../common_types'; import { Monitor, MonitorConfig } from './monitor'; export type JourneyOptions = { name: string; id?: string; tags?: string[]; + matrix?: Matrix; }; type HookType = 'before' | 'after'; @@ -61,11 +62,13 @@ export class Journey { steps: Step[] = []; hooks: Hooks = { before: [], after: [] }; monitor: Monitor; + params: Params = {}; constructor( options: JourneyOptions, callback: JourneyCallback, - location?: Location + location?: Location, + params?: Params, ) { this.name = options.name; this.id = options.id || options.name; @@ -73,6 +76,7 @@ export class Journey { this.callback = callback; this.location = location; this.updateMonitor({}); + this.params = params; } addStep(name: string, callback: VoidCallback, location?: Location) { diff --git a/src/matrix.ts b/src/matrix.ts new file mode 100644 index 00000000..5d674266 --- /dev/null +++ b/src/matrix.ts @@ -0,0 +1,96 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +import { createHash } from 'crypto'; +import type { Matrix } from './common_types'; + +export const getCombinations = (matrix: Matrix): Array> => { + const { values, adjustments } = matrix; + const matrixKeys = Object.keys(matrix.values); + const entries = Object.values(values); + + let combinations = calculateCombinations(entries[0]); + for (let i = 1; i < entries.length; i++) { + combinations = calculateCombinations(combinations, entries[i]); + } + + const matrixParams = combinations.map(combination => { + return getCombinationParams(matrixKeys, combination); + }); + + if (!adjustments) { + return matrixParams; + } + + const currentCombinations = new Set(matrixParams.map(params => { + const hash = createHash('sha256'); + const paramHash = hash.update(JSON.stringify(params)).digest('base64'); + return paramHash; + })); + + adjustments.forEach(adjustment => { + const hash = createHash('sha256'); + const adjustmentHash = hash.update(JSON.stringify(adjustment)).digest('base64'); + if (!currentCombinations.has(adjustmentHash)) { + matrixParams.push(adjustment); + } + }); + + return matrixParams; +} + +export const calculateCombinations = (groupA: Array, groupB?: Array): Array => { + const results = []; + groupA.forEach(optionA => { + if (!groupB) { + results.push([optionA]); + return; + } + groupB.forEach(optionB => { + if (Array.isArray(optionA)) { + return results.push([...optionA, optionB]) + } else { + return results.push([optionA, optionB]) + } + }); + }); + return results; +} + +export const getCombinationName = (name: string, combinations: Record) => { + const values = Object.values(combinations); + return values.reduce((acc, combination) => { + const nameAdjustment = typeof combination === 'object' ? JSON.stringify(combination) : combination.toString(); + acc += ` - ${nameAdjustment.toString()}`; + return acc; + }, name).trim(); +} + +export const getCombinationParams = (keys: string[], values: unknown[]): Record => { + return keys.reduce>((acc, key, index) => { + acc[key] = values[index]; + return acc; + }, {}); +} diff --git a/src/options.ts b/src/options.ts index f9549437..f6241529 100644 --- a/src/options.ts +++ b/src/options.ts @@ -100,6 +100,11 @@ export function normalizeOptions(cliArgs: CliArgs): RunOptions { */ options.params = Object.freeze(merge(config.params, cliArgs.params || {})); + /** + * Grab matrix only from config and not cliArgs + */ + options.matrix = Object.freeze(config.matrix); + /** * Merge playwright options from CLI and Synthetics config * and prefer individual options over other option From 03897ef61835f6f029e853534a24de498f0e1f5e Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Wed, 15 Feb 2023 15:42:26 -0500 Subject: [PATCH 2/5] remove logic from wrapFnWithLocation fn and add Suite abstraction --- src/core/index.ts | 33 +++++++--------------------- src/core/runner.ts | 51 +++++++++++++++++++++++++++++++++++++++++-- src/dsl/journey.ts | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 27 deletions(-) diff --git a/src/core/index.ts b/src/core/index.ts index ac4fa7ce..208d6d12 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -22,12 +22,10 @@ * THE SOFTWARE. * */ -import { Journey, JourneyCallback, JourneyOptions } from '../dsl'; +import { Journey, JourneyCallback, JourneyOptions, Suite } from '../dsl'; import Runner from './runner'; import { VoidCallback, HooksCallback, Location } from '../common_types'; -import { normalizeOptions } from '../options'; import { wrapFnWithLocation } from '../helpers'; -import { getCombinations, getCombinationName } from '../matrix'; import { log } from './logger'; import { MonitorConfig } from '../dsl/monitor'; @@ -35,7 +33,7 @@ import { MonitorConfig } from '../dsl/monitor'; /* TODO: Testing * Local vs global matrix: Local matrix fully overwrites global matrix, rather than merging * Adjustments: Duplicates in adjustments do not run extra journeys - * + * Regular params are combina /** * Use a gloabl Runner which would be accessed by the runtime and @@ -57,27 +55,12 @@ export const journey = wrapFnWithLocation( if (typeof options === 'string') { options = { name: options, id: options }; } - const { matrix: globalMatrix } = normalizeOptions({}); - const { matrix: journeyMatrix } = options; - if (!globalMatrix && !journeyMatrix) { - const j = new Journey(options, callback, location); - runner.addJourney(j); - return j; - } - - // local journey matrix takes priority over global matrix - const matrix = journeyMatrix || globalMatrix; - - if (!matrix.values) { - throw new Error('Please specify values for your testing matrix'); - } - - const combinations = getCombinations(matrix); - combinations.forEach(matrixParams => { - const name = getCombinationName((options as JourneyOptions)?.name, matrixParams); - const j = new Journey({...options as JourneyOptions, name}, callback, location, matrixParams); - runner.addJourney(j); - }) + const suite = new Suite(location); + const j = new Journey(options, callback, location); + suite.addJourney(j); + runner.addJourney(j); + runner.addSuite(suite); + return j; } ); diff --git a/src/core/runner.ts b/src/core/runner.ts index 30dba2e3..11b308af 100644 --- a/src/core/runner.ts +++ b/src/core/runner.ts @@ -25,7 +25,7 @@ import { join } from 'path'; import { mkdir, rm, writeFile } from 'fs/promises'; -import { Journey } from '../dsl/journey'; +import { Journey, Suite } from '../dsl/journey'; import { Step } from '../dsl/step'; import { reporters, Reporter } from '../reporters'; import { @@ -35,6 +35,7 @@ import { runParallel, generateUniqueId, } from '../helpers'; +import { getCombinationName, getCombinations } from '../matrix'; import { HooksCallback, Params, @@ -44,6 +45,7 @@ import { RunOptions, JourneyResult, StepResult, + Location, } from '../common_types'; import { PluginManager } from '../plugins'; import { PerformanceManager } from '../plugins'; @@ -68,6 +70,7 @@ export default class Runner { #reporter: Reporter; #currentJourney?: Journey = null; journeys: Journey[] = []; + suites: Map = new Map(); hooks: SuiteHooks = { beforeAll: [], afterAll: [] }; hookError: Error | undefined; monitor?: Monitor; @@ -142,6 +145,10 @@ export default class Runner { this.#currentJourney = journey; } + addSuite(suite: Suite) { + this.suites.set(suite.location, suite); + } + setReporter(options: RunOptions) { /** * Set up the corresponding reporter and fallback @@ -421,6 +428,38 @@ export default class Runner { return monitors; } + async parseMatrix(options: RunOptions) { + this.journeys.forEach(journey => { + const { matrix: globalMatrix } = options; + const { matrix: localMatrix } = journey; + // local journey matrix takes priority over global matrix + const matrix = localMatrix || globalMatrix; + + if (!matrix) { + return; + } + + if (!matrix.values) { + throw new Error('Please specify values for your testing matrix'); + } + + const suite = this.suites.get(journey.location); + suite.clearJourneys(); + + const combinations = getCombinations(matrix); + combinations.forEach(matrixParams => { + const j = journey.clone(); + const name = getCombinationName(j.name, matrixParams); + j.name = name; + j.id = name; + j.params = matrixParams; + this.addJourney(j); + suite.addJourney(j); + }); + }) + + } + async run(options: RunOptions) { const result: RunResult = {}; if (this.#active) { @@ -435,7 +474,15 @@ export default class Runner { }).catch(e => (this.hookError = e)); const { dryRun, match, tags } = options; - for (const journey of this.journeys) { + + this.parseMatrix(options); + + const journeys = Array.from(this.suites.values()).reduce((acc, suite) => { + const suiteJourneys = suite.entries; + return [...acc, ...suiteJourneys]; + }, []); + + for (const journey of journeys) { /** * Used by heartbeat to gather all registered journeys */ diff --git a/src/dsl/journey.ts b/src/dsl/journey.ts index 29c4dff0..ad52125b 100644 --- a/src/dsl/journey.ts +++ b/src/dsl/journey.ts @@ -57,12 +57,14 @@ export class Journey { name: string; id?: string; tags?: string[]; + matrix: Matrix; callback: JourneyCallback; location?: Location; steps: Step[] = []; hooks: Hooks = { before: [], after: [] }; monitor: Monitor; params: Params = {}; + parent!: Suite; constructor( options: JourneyOptions, @@ -73,6 +75,7 @@ export class Journey { this.name = options.name; this.id = options.id || options.name; this.tags = options.tags; + this.matrix = options.matrix; this.callback = callback; this.location = location; this.updateMonitor({}); @@ -124,4 +127,55 @@ export class Journey { const matchess = micromatch(this.tags || ['*'], pattern); return matchess.length > 0; } + + private _serialize(): any { + return { + options: { + name: this.name, + id: this.id, + tags: this.tags, + }, + callback: this.callback, + location: this.location, + steps: this.steps, + hooks: this.hooks, + monitor: this.monitor, + params: this.params, + }; + } + + clone(): Journey { + const data = this._serialize(); + const test = Journey._parse(data); + return test; + } + + static _parse(data: any): Journey { + const journey = new Journey(data.options, data.callback, data.location, data.params); + return journey; + } +} + +export class Suite { + location: Location; + private _entries: Journey[] = []; + + constructor( + location: Location, + ) { + this.location = location + } + + get entries() { + return this._entries; + } + + addJourney(journey: Journey) { + journey.parent = this; + this._entries.push(journey); + } + + clearJourneys() { + this._entries = []; + } } From c112b3c640286129255aed7b585398aed5dd316f Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Wed, 15 Feb 2023 16:15:04 -0500 Subject: [PATCH 3/5] handle project monitors --- src/core/index.ts | 8 +++----- src/core/runner.ts | 43 ++++++++++++++++++++++++++++++++++--------- src/loader.ts | 1 + 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/core/index.ts b/src/core/index.ts index 208d6d12..4151d5cd 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -22,7 +22,7 @@ * THE SOFTWARE. * */ -import { Journey, JourneyCallback, JourneyOptions, Suite } from '../dsl'; +import { Journey, JourneyCallback, JourneyOptions } from '../dsl'; import Runner from './runner'; import { VoidCallback, HooksCallback, Location } from '../common_types'; import { wrapFnWithLocation } from '../helpers'; @@ -33,7 +33,8 @@ import { MonitorConfig } from '../dsl/monitor'; /* TODO: Testing * Local vs global matrix: Local matrix fully overwrites global matrix, rather than merging * Adjustments: Duplicates in adjustments do not run extra journeys - * Regular params are combina + * Regular params are combined with matrix params + * Project monitors: name and id are overwritten only for matrix monitors /** * Use a gloabl Runner which would be accessed by the runtime and @@ -55,11 +56,8 @@ export const journey = wrapFnWithLocation( if (typeof options === 'string') { options = { name: options, id: options }; } - const suite = new Suite(location); const j = new Journey(options, callback, location); - suite.addJourney(j); runner.addJourney(j); - runner.addSuite(suite); return j; } ); diff --git a/src/core/runner.ts b/src/core/runner.ts index 11b308af..6c47512e 100644 --- a/src/core/runner.ts +++ b/src/core/runner.ts @@ -141,6 +141,14 @@ export default class Runner { } addJourney(journey: Journey) { + const journeySuite = this.suites.get(journey.location); + if (journeySuite) { + journeySuite.addJourney(journey); + } else { + const suite = new Suite(journey.location); + suite.addJourney(journey); + this.addSuite(suite); + } this.journeys.push(journey); this.#currentJourney = journey; } @@ -392,10 +400,13 @@ export default class Runner { } buildMonitors(options: RunOptions) { + /* Build out monitors according to matrix specs */ + this.parseMatrix(options); + /** * Update the global monitor configuration required for * setting defaults - */ + */ this.updateMonitor({ throttling: options.throttling, schedule: options.schedule, @@ -407,7 +418,10 @@ export default class Runner { const { match, tags } = options; const monitors: Monitor[] = []; - for (const journey of this.journeys) { + + const journeys = this.getAllJourneys(); + + for (const journey of journeys) { const params = Object.freeze({ ...this.monitor.config?.params, ...options.params, ...journey.params }); if (!journey.isMatch(match, tags)) { continue; @@ -419,9 +433,15 @@ export default class Runner { */ journey.callback({ params: params } as any); journey.monitor.update({ - ...this.monitor?.config, + ...this.monitor?.config, params: Object.keys(params).length ? params : undefined }); + + /* Only overwrite name and id values when using matrix */ + if (journey.matrix) { + journey.monitor.config.name = journey.name; + journey.monitor.config.id = journey.id; + } journey.monitor.validate(); monitors.push(journey.monitor); } @@ -453,11 +473,19 @@ export default class Runner { j.name = name; j.id = name; j.params = matrixParams; + j.matrix = matrix; this.addJourney(j); suite.addJourney(j); }); - }) - + }) + } + + getAllJourneys() { + const journeys = Array.from(this.suites.values()).reduce((acc, suite) => { + const suiteJourneys = suite.entries; + return [...acc, ...suiteJourneys]; + }, []); + return journeys; } async run(options: RunOptions) { @@ -477,10 +505,7 @@ export default class Runner { this.parseMatrix(options); - const journeys = Array.from(this.suites.values()).reduce((acc, suite) => { - const suiteJourneys = suite.entries; - return [...acc, ...suiteJourneys]; - }, []); + const journeys = this.getAllJourneys(); for (const journey of journeys) { /** diff --git a/src/loader.ts b/src/loader.ts index ec378308..2a3ad617 100644 --- a/src/loader.ts +++ b/src/loader.ts @@ -71,6 +71,7 @@ export async function loadTestFiles(options: CliArgs, args: string[]) { loadInlineScript(source); return; } + /** * Handle piped files by reading the STDIN * ex: ls example/suites/*.js | npx @elastic/synthetics From b6cb50e164bd90ae33c8adcdcf93303fa883dfc8 Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Thu, 16 Feb 2023 13:34:23 -0500 Subject: [PATCH 4/5] remove values and prefer adjustments only - add support for playwright options --- __tests__/cli.test.ts | 33 ++++++++--- __tests__/fixtures/matrix/example.journey.ts | 33 +++++++++++ .../fixtures/matrix/synthetics.config.ts | 31 ++++++++++ src/common_types.ts | 4 +- src/core/index.ts | 3 + src/core/runner.ts | 29 ++++++--- src/dsl/journey.ts | 3 +- src/matrix.ts | 59 ++++++++++--------- 8 files changed, 148 insertions(+), 47 deletions(-) create mode 100644 __tests__/fixtures/matrix/example.journey.ts create mode 100644 __tests__/fixtures/matrix/synthetics.config.ts diff --git a/__tests__/cli.test.ts b/__tests__/cli.test.ts index 560eb8c6..d0260b0f 100644 --- a/__tests__/cli.test.ts +++ b/__tests__/cli.test.ts @@ -34,7 +34,7 @@ import { safeNDJSONParse, } from '../src/helpers'; -describe('CLI', () => { +describe.only('CLI', () => { let server: Server; let serverParams: { url: string }; beforeAll(async () => { @@ -44,6 +44,7 @@ describe('CLI', () => { afterAll(async () => await server.close()); const FIXTURES_DIR = join(__dirname, 'fixtures'); + const MATRIX_FIXTURES_DIR = join(__dirname, 'fixtures/matrix'); // jest by default sets NODE_ENV to `test` const originalNodeEnv = process.env['NODE_ENV']; @@ -541,7 +542,7 @@ describe('CLI', () => { }); }); - describe('playwright options', () => { + describe.only('playwright options', () => { it('pass playwright options to runner', async () => { const cli = new CLIMock() .args([ @@ -574,12 +575,30 @@ describe('CLI', () => { }), ]) .run(); - await cli.waitFor('step/end'); - const output = cli.output(); + await cli.waitFor('step/end'); + const output = cli.output(); + expect(await cli.exitCode).toBe(1); + expect(JSON.parse(output).step).toMatchObject({ + status: 'failed', + }); + }); + + it.only('handles matrix', async () => { + const cli = new CLIMock() + .args([ + join(MATRIX_FIXTURES_DIR, 'example.journey.ts'), + '--reporter', + 'json', + '--config', + join(MATRIX_FIXTURES_DIR, 'synthetics.config.ts'), + ]) + .run(); expect(await cli.exitCode).toBe(1); - expect(JSON.parse(output).step).toMatchObject({ - status: 'failed', - }); + const endEvents = cli.buffer().filter(data => data.includes('journey/end')); + const badsslFailing = endEvents.find(event => event.includes('badssl failing')); + const badsslSucceeded = endEvents.find(event => event.includes('badssl passing')) + expect(JSON.parse(badsslFailing || '')?.journey?.status).toBe("failed"); + expect(JSON.parse(badsslSucceeded || '')?.journey?.status).toBe("succeeded"); }); }); }); diff --git a/__tests__/fixtures/matrix/example.journey.ts b/__tests__/fixtures/matrix/example.journey.ts new file mode 100644 index 00000000..57ee0e34 --- /dev/null +++ b/__tests__/fixtures/matrix/example.journey.ts @@ -0,0 +1,33 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +import { journey, step } from '../../../src/index'; + +journey('matrix journey', ({ page, params }) => { + step('go to test page', async () => { + await page.goto(params.url, { waitUntil: 'networkidle' }); + await page.waitForSelector(`text=${params.assertion}`, { timeout: 1500 }); + }); +}); diff --git a/__tests__/fixtures/matrix/synthetics.config.ts b/__tests__/fixtures/matrix/synthetics.config.ts new file mode 100644 index 00000000..ef008592 --- /dev/null +++ b/__tests__/fixtures/matrix/synthetics.config.ts @@ -0,0 +1,31 @@ +import type { SyntheticsConfig } from '../../../src'; + +module.exports = () => { + const config: SyntheticsConfig = { + params: { + url: 'dev', + }, + matrix: { + // values: { + // url: ['https://elastic.github.io/synthetics-demo/'], + // assertion: ['test', 'test-2'] + // } + adjustments: [{ + name: 'badssl failing', + url: 'https://expired.badssl.com/', + assertion: 'expired', + playwrightOptions: { + ignoreHTTPSErrors: false, + } + }, { + name: 'badssl passing', + url: 'https://expired.badssl.com/', + assertion: 'expired', + playwrightOptions: { + ignoreHTTPSErrors: true, + } + }] + } + }; + return config; +}; diff --git a/src/common_types.ts b/src/common_types.ts index 2b6413f4..6ca9f60a 100644 --- a/src/common_types.ts +++ b/src/common_types.ts @@ -256,8 +256,8 @@ export type ProjectSettings = { }; export type Matrix = { - values: Record; - adjustments?: Array>; + // values: Record; + adjustments: Array & { playwrightOptions?: PlaywrightOptions, name: string }>; } export type PlaywrightOptions = LaunchOptions & BrowserContextOptions; diff --git a/src/core/index.ts b/src/core/index.ts index 4151d5cd..f41f5ce0 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -35,6 +35,9 @@ import { MonitorConfig } from '../dsl/monitor'; * Adjustments: Duplicates in adjustments do not run extra journeys * Regular params are combined with matrix params * Project monitors: name and id are overwritten only for matrix monitors + * How does monitor params/playwright options interact with matrix overrides? + * Should it be global params -> monitor params -> matrix params + * Should it be global playwrightOptions -> monitor playWrightOptions -> matrix playwrightOptions /** * Use a gloabl Runner which would be accessed by the runtime and diff --git a/src/core/runner.ts b/src/core/runner.ts index 6c47512e..50e42f96 100644 --- a/src/core/runner.ts +++ b/src/core/runner.ts @@ -35,7 +35,7 @@ import { runParallel, generateUniqueId, } from '../helpers'; -import { getCombinationName, getCombinations } from '../matrix'; +import { getCombinations } from '../matrix'; import { HooksCallback, Params, @@ -355,7 +355,8 @@ export default class Runner { async runJourney(journey: Journey, options: RunOptions) { const result: JourneyResult = { status: 'succeeded' }; const params = Object.freeze({...options.params, ...journey.params}); - const journeyOptions = { ...options, params}; + const playwrightOptions = { ...options.playwrightOptions, ...journey.playwrightOptions }; + const journeyOptions = { ...options, params, playwrightOptions }; const context = await Runner.createContext(journeyOptions); log(`Runner: start journey (${journey.name})`); try { @@ -422,7 +423,8 @@ export default class Runner { const journeys = this.getAllJourneys(); for (const journey of journeys) { - const params = Object.freeze({ ...this.monitor.config?.params, ...options.params, ...journey.params }); + const params = Object.freeze({ ...options.params, ...this.monitor.config?.params, ...journey.params }); + const playwrightOptions = { ...options.playwrightOptions, ...this.monitor.config?.playwrightOptions, ...journey.playwrightOptions } if (!journey.isMatch(match, tags)) { continue; } @@ -434,13 +436,15 @@ export default class Runner { journey.callback({ params: params } as any); journey.monitor.update({ ...this.monitor?.config, - params: Object.keys(params).length ? params : undefined + params: Object.keys(params).length ? params : undefined, + playwrightOptions }); /* Only overwrite name and id values when using matrix */ if (journey.matrix) { journey.monitor.config.name = journey.name; journey.monitor.config.id = journey.id; + journey.monitor.config.playwrightOptions = playwrightOptions; } journey.monitor.validate(); monitors.push(journey.monitor); @@ -459,8 +463,13 @@ export default class Runner { return; } - if (!matrix.values) { - throw new Error('Please specify values for your testing matrix'); + if (!matrix.adjustments) { + throw new Error('Please specify adjustments for your testing matrix'); + } + + + if (matrix.adjustments.some(adjustment => !adjustment.name)) { + throw new Error('Please specify a name for each adjustment'); } const suite = this.suites.get(journey.location); @@ -469,13 +478,15 @@ export default class Runner { const combinations = getCombinations(matrix); combinations.forEach(matrixParams => { const j = journey.clone(); - const name = getCombinationName(j.name, matrixParams); + const { playwrightOptions, name, ...params } = matrixParams; + if (playwrightOptions) { + j.playwrightOptions = { ...options.playwrightOptions, ...playwrightOptions } + } j.name = name; j.id = name; - j.params = matrixParams; + j.params = params; j.matrix = matrix; this.addJourney(j); - suite.addJourney(j); }); }) } diff --git a/src/dsl/journey.ts b/src/dsl/journey.ts index ad52125b..6283d4e3 100644 --- a/src/dsl/journey.ts +++ b/src/dsl/journey.ts @@ -32,7 +32,7 @@ import { } from 'playwright-chromium'; import micromatch, { isMatch } from 'micromatch'; import { Step } from './step'; -import { VoidCallback, HooksCallback, Params, Location, Matrix } from '../common_types'; +import { VoidCallback, HooksCallback, Params, Location, Matrix, PlaywrightOptions } from '../common_types'; import { Monitor, MonitorConfig } from './monitor'; export type JourneyOptions = { @@ -65,6 +65,7 @@ export class Journey { monitor: Monitor; params: Params = {}; parent!: Suite; + playwrightOptions: PlaywrightOptions = {}; constructor( options: JourneyOptions, diff --git a/src/matrix.ts b/src/matrix.ts index 5d674266..108ecbeb 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -23,42 +23,45 @@ * */ -import { createHash } from 'crypto'; +// import { createHash } from 'crypto'; import type { Matrix } from './common_types'; -export const getCombinations = (matrix: Matrix): Array> => { - const { values, adjustments } = matrix; - const matrixKeys = Object.keys(matrix.values); - const entries = Object.values(values); +export const getCombinations = (matrix: Matrix): Matrix['adjustments'] => { + const { adjustments } = matrix; - let combinations = calculateCombinations(entries[0]); - for (let i = 1; i < entries.length; i++) { - combinations = calculateCombinations(combinations, entries[i]); - } + // no longer need this logic when removing values - const matrixParams = combinations.map(combination => { - return getCombinationParams(matrixKeys, combination); - }); + // const matrixKeys = Object.keys(matrix.values); + // const entries = Object.values(values); - if (!adjustments) { - return matrixParams; - } + // let combinations = calculateCombinations(entries[0]); + // for (let i = 1; i < entries.length; i++) { + // combinations = calculateCombinations(combinations, entries[i]); + // } - const currentCombinations = new Set(matrixParams.map(params => { - const hash = createHash('sha256'); - const paramHash = hash.update(JSON.stringify(params)).digest('base64'); - return paramHash; - })); + // const matrixParams = combinations.map(combination => { + // return getCombinationParams(matrixKeys, combination); + // }); - adjustments.forEach(adjustment => { - const hash = createHash('sha256'); - const adjustmentHash = hash.update(JSON.stringify(adjustment)).digest('base64'); - if (!currentCombinations.has(adjustmentHash)) { - matrixParams.push(adjustment); - } - }); + // if (!adjustments) { + // return matrixParams; + // } + + // const currentCombinations = new Set(matrixParams.map(params => { + // const hash = createHash('sha256'); + // const paramHash = hash.update(JSON.stringify(params)).digest('base64'); + // return paramHash; + // })); + + // adjustments.forEach(adjustment => { + // const hash = createHash('sha256'); + // const adjustmentHash = hash.update(JSON.stringify(adjustment)).digest('base64'); + // if (!currentCombinations.has(adjustmentHash)) { + // matrixParams.push(adjustment); + // } + // }); - return matrixParams; + return adjustments; } export const calculateCombinations = (groupA: Array, groupB?: Array): Array => { From f842db39e85b2ec166c47b9c242366f2d0c82cf8 Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Mon, 6 Mar 2023 17:01:35 -0500 Subject: [PATCH 5/5] adjust types for matrix --- __tests__/fixtures/matrix/synthetics.config.ts | 16 ++++++++-------- src/common_types.ts | 2 +- src/core/runner.ts | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/__tests__/fixtures/matrix/synthetics.config.ts b/__tests__/fixtures/matrix/synthetics.config.ts index ef008592..a4fb9473 100644 --- a/__tests__/fixtures/matrix/synthetics.config.ts +++ b/__tests__/fixtures/matrix/synthetics.config.ts @@ -6,21 +6,21 @@ module.exports = () => { url: 'dev', }, matrix: { - // values: { - // url: ['https://elastic.github.io/synthetics-demo/'], - // assertion: ['test', 'test-2'] - // } adjustments: [{ name: 'badssl failing', - url: 'https://expired.badssl.com/', - assertion: 'expired', + params: { + url: 'https://expired.badssl.com/', + assertion: 'expired', + }, playwrightOptions: { ignoreHTTPSErrors: false, } }, { name: 'badssl passing', - url: 'https://expired.badssl.com/', - assertion: 'expired', + params: { + url: 'https://expired.badssl.com/', + assertion: 'expired', + }, playwrightOptions: { ignoreHTTPSErrors: true, } diff --git a/src/common_types.ts b/src/common_types.ts index 6ca9f60a..77270949 100644 --- a/src/common_types.ts +++ b/src/common_types.ts @@ -257,7 +257,7 @@ export type ProjectSettings = { export type Matrix = { // values: Record; - adjustments: Array & { playwrightOptions?: PlaywrightOptions, name: string }>; + adjustments: Array<{ playwrightOptions?: PlaywrightOptions, name: string, params?: Record }>; } export type PlaywrightOptions = LaunchOptions & BrowserContextOptions; diff --git a/src/core/runner.ts b/src/core/runner.ts index 50e42f96..04f71b27 100644 --- a/src/core/runner.ts +++ b/src/core/runner.ts @@ -478,12 +478,12 @@ export default class Runner { const combinations = getCombinations(matrix); combinations.forEach(matrixParams => { const j = journey.clone(); - const { playwrightOptions, name, ...params } = matrixParams; + const { playwrightOptions, name, params } = matrixParams; if (playwrightOptions) { j.playwrightOptions = { ...options.playwrightOptions, ...playwrightOptions } } - j.name = name; - j.id = name; + j.name = `${j.name} - ${name}`; + j.id = `${j.id}-${name}`; j.params = params; j.matrix = matrix; this.addJourney(j);