From d5bfecfbdcdc61e0655ed307f92c859d7bdd4dee Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Mon, 13 Feb 2023 16:25:25 -0500 Subject: [PATCH] add basic support for testing matrix --- src/common_types.ts | 7 ++++ src/core/index.ts | 34 +++++++++++++--- src/core/runner.ts | 15 ++++--- src/dsl/journey.ts | 8 +++- src/matrix.ts | 96 +++++++++++++++++++++++++++++++++++++++++++++ src/options.ts | 5 +++ 6 files changed, 152 insertions(+), 13 deletions(-) create mode 100644 src/matrix.ts diff --git a/src/common_types.ts b/src/common_types.ts index ce3ddd2f..2b6413f4 100644 --- a/src/common_types.ts +++ b/src/common_types.ts @@ -237,6 +237,7 @@ export type RunOptions = BaseArgs & { environment?: string; playwrightOptions?: PlaywrightOptions; networkConditions?: NetworkConditions; + matrix?: Matrix; reporter?: BuiltInReporterName | ReporterInstance; }; @@ -254,12 +255,18 @@ export type ProjectSettings = { space: string; }; +export type Matrix = { + values: Record; + adjustments?: Array>; +} + export type PlaywrightOptions = LaunchOptions & BrowserContextOptions; export type SyntheticsConfig = { params?: Params; playwrightOptions?: PlaywrightOptions; monitor?: MonitorConfig; project?: ProjectSettings; + matrix?: Matrix; }; /** Runner Payload types */ diff --git a/src/core/index.ts b/src/core/index.ts index ce806e6a..ac4fa7ce 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -22,14 +22,21 @@ * THE SOFTWARE. * */ - import { Journey, JourneyCallback, JourneyOptions } from '../dsl'; import Runner from './runner'; import { VoidCallback, HooksCallback, Location } from '../common_types'; +import { normalizeOptions } from '../options'; import { wrapFnWithLocation } from '../helpers'; +import { getCombinations, getCombinationName } from '../matrix'; import { log } from './logger'; import { MonitorConfig } from '../dsl/monitor'; + +/* TODO: Testing + * Local vs global matrix: Local matrix fully overwrites global matrix, rather than merging + * Adjustments: Duplicates in adjustments do not run extra journeys + * + /** * Use a gloabl Runner which would be accessed by the runtime and * required to handle the local vs global invocation through CLI @@ -38,7 +45,6 @@ const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER'); if (!global[SYNTHETICS_RUNNER]) { global[SYNTHETICS_RUNNER] = new Runner(); } - export const runner: Runner = global[SYNTHETICS_RUNNER]; export const journey = wrapFnWithLocation( @@ -51,9 +57,27 @@ export const journey = wrapFnWithLocation( if (typeof options === 'string') { options = { name: options, id: options }; } - const j = new Journey(options, callback, location); - runner.addJourney(j); - return j; + const { matrix: globalMatrix } = normalizeOptions({}); + const { matrix: journeyMatrix } = options; + if (!globalMatrix && !journeyMatrix) { + const j = new Journey(options, callback, location); + runner.addJourney(j); + return j; + } + + // local journey matrix takes priority over global matrix + const matrix = journeyMatrix || globalMatrix; + + if (!matrix.values) { + throw new Error('Please specify values for your testing matrix'); + } + + const combinations = getCombinations(matrix); + combinations.forEach(matrixParams => { + const name = getCombinationName((options as JourneyOptions)?.name, matrixParams); + const j = new Journey({...options as JourneyOptions, name}, callback, location, matrixParams); + runner.addJourney(j); + }) } ); diff --git a/src/core/runner.ts b/src/core/runner.ts index d3dee99e..2aaac169 100644 --- a/src/core/runner.ts +++ b/src/core/runner.ts @@ -339,16 +339,18 @@ export default class Runner { async runJourney(journey: Journey, options: RunOptions) { const result: JourneyResult = { status: 'succeeded' }; - const context = await Runner.createContext(options); + const params = {...options.params, ...journey.params}; + const journeyOptions = { ...options, params}; + const context = await Runner.createContext(journeyOptions); log(`Runner: start journey (${journey.name})`); try { this.registerJourney(journey, context); const hookArgs = { env: options.environment, - params: options.params, + params: params, }; await this.runBeforeHook(journey, hookArgs); - const stepResults = await this.runSteps(journey, context, options); + const stepResults = await this.runSteps(journey, context, journeyOptions); /** * Mark journey as failed if any intermediate step fails */ @@ -363,7 +365,7 @@ export default class Runner { result.status = 'failed'; result.error = e; } finally { - await this.endJourney(journey, { ...context, ...result }, options); + await this.endJourney(journey, { ...context, ...result }, journeyOptions); await Gatherer.dispose(context.driver); } log(`Runner: end journey (${journey.name})`); @@ -399,6 +401,7 @@ export default class Runner { const { match, tags } = options; const monitors: Monitor[] = []; for (const journey of this.journeys) { + const params = { ...this.monitor.config?.params, ...options.params, ...journey.params }; if (!journey.isMatch(match, tags)) { continue; } @@ -407,8 +410,8 @@ export default class Runner { * Execute dummy callback to get all monitor specific * configurations for the current journey */ - journey.callback({ params: options.params } as any); - journey.monitor.update(this.monitor?.config); + journey.callback({ params: params } as any); + journey.monitor.update({ ...this.monitor?.config, params }); journey.monitor.validate(); monitors.push(journey.monitor); } diff --git a/src/dsl/journey.ts b/src/dsl/journey.ts index b9bb7200..29c4dff0 100644 --- a/src/dsl/journey.ts +++ b/src/dsl/journey.ts @@ -32,13 +32,14 @@ import { } from 'playwright-chromium'; import micromatch, { isMatch } from 'micromatch'; import { Step } from './step'; -import { VoidCallback, HooksCallback, Params, Location } from '../common_types'; +import { VoidCallback, HooksCallback, Params, Location, Matrix } from '../common_types'; import { Monitor, MonitorConfig } from './monitor'; export type JourneyOptions = { name: string; id?: string; tags?: string[]; + matrix?: Matrix; }; type HookType = 'before' | 'after'; @@ -61,11 +62,13 @@ export class Journey { steps: Step[] = []; hooks: Hooks = { before: [], after: [] }; monitor: Monitor; + params: Params = {}; constructor( options: JourneyOptions, callback: JourneyCallback, - location?: Location + location?: Location, + params?: Params, ) { this.name = options.name; this.id = options.id || options.name; @@ -73,6 +76,7 @@ export class Journey { this.callback = callback; this.location = location; this.updateMonitor({}); + this.params = params; } addStep(name: string, callback: VoidCallback, location?: Location) { diff --git a/src/matrix.ts b/src/matrix.ts new file mode 100644 index 00000000..5d674266 --- /dev/null +++ b/src/matrix.ts @@ -0,0 +1,96 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +import { createHash } from 'crypto'; +import type { Matrix } from './common_types'; + +export const getCombinations = (matrix: Matrix): Array> => { + const { values, adjustments } = matrix; + const matrixKeys = Object.keys(matrix.values); + const entries = Object.values(values); + + let combinations = calculateCombinations(entries[0]); + for (let i = 1; i < entries.length; i++) { + combinations = calculateCombinations(combinations, entries[i]); + } + + const matrixParams = combinations.map(combination => { + return getCombinationParams(matrixKeys, combination); + }); + + if (!adjustments) { + return matrixParams; + } + + const currentCombinations = new Set(matrixParams.map(params => { + const hash = createHash('sha256'); + const paramHash = hash.update(JSON.stringify(params)).digest('base64'); + return paramHash; + })); + + adjustments.forEach(adjustment => { + const hash = createHash('sha256'); + const adjustmentHash = hash.update(JSON.stringify(adjustment)).digest('base64'); + if (!currentCombinations.has(adjustmentHash)) { + matrixParams.push(adjustment); + } + }); + + return matrixParams; +} + +export const calculateCombinations = (groupA: Array, groupB?: Array): Array => { + const results = []; + groupA.forEach(optionA => { + if (!groupB) { + results.push([optionA]); + return; + } + groupB.forEach(optionB => { + if (Array.isArray(optionA)) { + return results.push([...optionA, optionB]) + } else { + return results.push([optionA, optionB]) + } + }); + }); + return results; +} + +export const getCombinationName = (name: string, combinations: Record) => { + const values = Object.values(combinations); + return values.reduce((acc, combination) => { + const nameAdjustment = typeof combination === 'object' ? JSON.stringify(combination) : combination.toString(); + acc += ` - ${nameAdjustment.toString()}`; + return acc; + }, name).trim(); +} + +export const getCombinationParams = (keys: string[], values: unknown[]): Record => { + return keys.reduce>((acc, key, index) => { + acc[key] = values[index]; + return acc; + }, {}); +} diff --git a/src/options.ts b/src/options.ts index f9549437..f6241529 100644 --- a/src/options.ts +++ b/src/options.ts @@ -100,6 +100,11 @@ export function normalizeOptions(cliArgs: CliArgs): RunOptions { */ options.params = Object.freeze(merge(config.params, cliArgs.params || {})); + /** + * Grab matrix only from config and not cliArgs + */ + options.matrix = Object.freeze(config.matrix); + /** * Merge playwright options from CLI and Synthetics config * and prefer individual options over other option