add basic support for testing matrix

__tests__/cli.test.ts

@@ -34,7 +34,7 @@ import {
   safeNDJSONParse,
 } from '../src/helpers';
 
-describe('CLI', () => {
+describe.only('CLI', () => {
   let server: Server;
   let serverParams: { url: string };
   beforeAll(async () => {
@@ -44,6 +44,7 @@ describe('CLI', () => {
   afterAll(async () => await server.close());
 
   const FIXTURES_DIR = join(__dirname, 'fixtures');
+  const MATRIX_FIXTURES_DIR = join(__dirname, 'fixtures/matrix');
 
   // jest by default sets NODE_ENV to `test`
   const originalNodeEnv = process.env['NODE_ENV'];
@@ -541,7 +542,7 @@ describe('CLI', () => {
     });
   });
 
-  describe('playwright options', () => {
+  describe.only('playwright options', () => {
     it('pass playwright options to runner', async () => {
       const cli = new CLIMock()
         .args([
@@ -574,12 +575,30 @@ describe('CLI', () => {
           }),
         ])
         .run();
-      await cli.waitFor('step/end');
-      const output = cli.output();
+        await cli.waitFor('step/end');
+        const output = cli.output();
+        expect(await cli.exitCode).toBe(1);
+        expect(JSON.parse(output).step).toMatchObject({
+          status: 'failed',
+        });
+    });
+
+    it.only('handles matrix', async () => {
+      const cli = new CLIMock()
+        .args([
+          join(MATRIX_FIXTURES_DIR, 'example.journey.ts'),
+          '--reporter',
+          'json',
+          '--config',
+          join(MATRIX_FIXTURES_DIR, 'synthetics.config.ts'),
+        ])
+        .run();
       expect(await cli.exitCode).toBe(1);
-      expect(JSON.parse(output).step).toMatchObject({
-        status: 'failed',
-      });
+      const endEvents = cli.buffer().filter(data => data.includes('journey/end'));
+      const badsslFailing = endEvents.find(event => event.includes('badssl failing'));
+      const badsslSucceeded = endEvents.find(event => event.includes('badssl passing'))
+      expect(JSON.parse(badsslFailing || '')?.journey?.status).toBe("failed");
+      expect(JSON.parse(badsslSucceeded || '')?.journey?.status).toBe("succeeded");
     });
   });
 });

__tests__/core/runner.test.ts

@@ -729,7 +729,7 @@ describe('runner', () => {
       type: 'browser',
       tags: [],
       locations: ['united_kingdom'],
-      privaateLocations: undefined,
+      privateLocations: undefined,
       schedule: 3,
       params: undefined,
       playwrightOptions: undefined,

__tests__/fixtures/matrix/example.journey.ts

@@ -0,0 +1,33 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { journey, step } from '../../../src/index';
+
+journey('matrix journey', ({ page, params }) => {
+  step('go to test page', async () => {
+    await page.goto(params.url, { waitUntil: 'networkidle' });
+    await page.waitForSelector(`text=${params.assertion}`, { timeout: 1500 });
+  });
+});

__tests__/fixtures/matrix/synthetics.config.ts

@@ -0,0 +1,31 @@
+import type { SyntheticsConfig } from '../../../src';
+
+module.exports = () => {
+  const config: SyntheticsConfig = {
+    params: {
+      url: 'dev',
+    },
+    matrix: {
+      adjustments: [{
+        name: 'badssl failing',
+        params: {
+          url: 'https://expired.badssl.com/',
+          assertion: 'expired',
+        },
+        playwrightOptions: {
+          ignoreHTTPSErrors: false,
+        }
+      }, {
+        name: 'badssl passing',
+        params: {
+          url: 'https://expired.badssl.com/',
+          assertion: 'expired',
+        },
+        playwrightOptions: {
+          ignoreHTTPSErrors: true,
+        }
+      }]
+    }
+  };
+  return config;
+};

src/common_types.ts

@@ -237,6 +237,7 @@ export type RunOptions = BaseArgs & {
   environment?: string;
   playwrightOptions?: PlaywrightOptions;
   networkConditions?: NetworkConditions;
+  matrix?: Matrix;
   reporter?: BuiltInReporterName | ReporterInstance;
 };
 
@@ -254,12 +255,18 @@ export type ProjectSettings = {
   space: string;
 };
 
+export type Matrix = {
+  // values: Record<string, unknown[]>;
+  adjustments: Array<{ playwrightOptions?: PlaywrightOptions, name: string, params?: Record<string, unknown> }>;
+}
+
 export type PlaywrightOptions = LaunchOptions & BrowserContextOptions;
 export type SyntheticsConfig = {
   params?: Params;
   playwrightOptions?: PlaywrightOptions;
   monitor?: MonitorConfig;
   project?: ProjectSettings;
+  matrix?: Matrix;
 };
 
 /** Runner Payload types */

src/core/index.ts

@@ -22,14 +22,23 @@
  * THE SOFTWARE.
  *
  */
-
 import { Journey, JourneyCallback, JourneyOptions } from '../dsl';
 import Runner from './runner';
 import { VoidCallback, HooksCallback, Location } from '../common_types';
 import { wrapFnWithLocation } from '../helpers';
 import { log } from './logger';
 import { MonitorConfig } from '../dsl/monitor';
 
+
+/* TODO: Testing
+ * Local vs global matrix: Local matrix fully overwrites global matrix, rather than merging
+ * Adjustments: Duplicates in adjustments do not run extra journeys
+ * Regular params are combined with matrix params
+ * Project monitors: name and id are overwritten only for matrix monitors
+ * How does monitor params/playwright options interact with matrix overrides? 
+ *      Should it be global params -> monitor params -> matrix params
+ *      Should it be global playwrightOptions -> monitor playWrightOptions -> matrix playwrightOptions
+
 /**
  * Use a gloabl Runner which would be accessed by the runtime and
  * required to handle the local vs global invocation through CLI
@@ -38,7 +47,6 @@ const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER');
 if (!global[SYNTHETICS_RUNNER]) {
   global[SYNTHETICS_RUNNER] = new Runner();
 }
-
 export const runner: Runner = global[SYNTHETICS_RUNNER];
 
 export const journey = wrapFnWithLocation(