Implement new upload options (#519)

* Implement new `upload` options * fix `maxAttempts` in Cypress * Tweak Cypress plugin options * remove inaccurate comment * tweak the playwright config * Create few-suits-arrive.md * add changeset for the old CLI * fmt * simplify advanced upload options decoding * add assert * add a comment
replayio · Jun 13, 2024 · 34b1ba7 · 34b1ba7
1 parent 81179b9
commit 34b1ba7
Show file tree

Hide file tree

Showing 11 changed files with 343 additions and 217 deletions.
diff --git a/.changeset/dirty-buckets-train.md b/.changeset/dirty-buckets-train.md
@@ -0,0 +1,5 @@
+---
+"@replayio/replay": patch
+---
+
+Export a new version of the new test run metadata validator
diff --git a/.changeset/few-suits-arrive.md b/.changeset/few-suits-arrive.md
@@ -0,0 +1,9 @@
+---
+"@replayio/playwright": patch
+"@replayio/test-utils": patch
+---
+
+Added new `upload` options:
+
+- `statusThreshold`: this accepts one of `'all'`, `'failed-and-flaky'` or `'failed'` and it's used to skip uploading recordings for test runs that don't satisfy the desired threshold (eg. it allows to filter out recordings of passed tests)
+- `minimizeUploads`: a boolean flag that helps to minimize the amount of uploaded recordings. With this flag a minimal set of recordings associated with a retried test is uploaded
diff --git a/examples/create-react-app-typescript/.gitignore b/examples/create-react-app-typescript/.gitignore
@@ -13,6 +13,7 @@
 
 # testing
 /coverage
+/test-results
 
 # production
 /build

diff --git a/examples/create-react-app-typescript/playwright.config.ts b/examples/create-react-app-typescript/playwright.config.ts
@@ -1,5 +1,5 @@
 import { defineConfig } from "@playwright/test";
-import { devices as replayDevices } from "@replayio/playwright";
+import { devices as replayDevices, replayReporter } from "@replayio/playwright";
 
 export default defineConfig({
   forbidOnly: !!process.env.CI,
@@ -14,9 +14,9 @@ export default defineConfig({
     reuseExistingServer: !process.env.CI,
   },
   reporter: [
-    ["@replayio/playwright/reporter"],
     // replicating Playwright's defaults
     process.env.CI ? (["dot"] as const) : (["list"] as const),
+    replayReporter({}),
   ],
   projects: [
     {

diff --git a/packages/cypress/src/reporter.ts b/packages/cypress/src/reporter.ts
@@ -22,7 +22,13 @@ type ReplayCypressRecordingMetadata = {
   test: TestMetadataV2.TestRun;
 };
 
-export interface PluginOptions extends ReplayReporterConfig<ReplayCypressRecordingMetadata> {}
+// `filter` is re-applied here so its deprecated comment gets lost here
+// `upload` gets simplified to a boolean (until the advanced options are tested with Cypress)
+export interface PluginOptions
+  extends Omit<ReplayReporterConfig<ReplayCypressRecordingMetadata>, "filter" | "upload"> {
+  filter?: ReplayReporterConfig<ReplayCypressRecordingMetadata>["filter"];
+  upload?: boolean;
+}
 
 const debug = dbg("replay:cypress:reporter");
 const MAX_WAIT = 20_000;
@@ -168,6 +174,7 @@ class CypressReporter {
       },
       result: "unknown",
       attempt: 1,
+      maxAttempts: 1,
       events: {
         afterAll: [],
         afterEach: [],

diff --git a/packages/cypress/src/steps.ts b/packages/cypress/src/steps.ts
@@ -63,15 +63,13 @@ function getTestsFromResults(
 ) {
   const startEvents = sortSteps(testStartSteps);
 
-  function getIdForTest(result: CypressCommandLine.TestResult) {
+  function getStartTestStep(result: CypressCommandLine.TestResult) {
     const startEventIndex = startEvents.findIndex(
       e => e.test.every((t, i) => t === result.title[i]) && e.test.length === result.title.length
     );
     if (startEventIndex !== -1) {
       const startEvent = startEvents.splice(startEventIndex, 1)[0];
-      if (startEvent.testId != null) {
-        return startEvent.testId;
-      }
+      return startEvent;
     }
   }
 
@@ -80,39 +78,43 @@ function getTestsFromResults(
     const title = scope.pop()!;
     const lastAttemptIndex = result.attempts.length - 1;
 
-    return result.attempts.map<Test>((a, attemptIndex) => ({
-      id: getIdForTest(result) ?? attemptIndex,
-      // those properties don't exist since Cypress 13: https://github.com/cypress-io/cypress/pull/27230
-      // TODO: remove it in PRO-640
-      approximateDuration: (a as any).duration || (a as any).wallClockDuration || 0,
-      attempt: attemptIndex + 1,
-      source: {
-        title,
-        scope,
-      },
-      result: mapStateToResult(a.state),
-      events: {
-        beforeAll: [],
-        afterAll: [],
-        beforeEach: [],
-        afterEach: [],
-        main: [],
-      },
-      // attempt.error is available here:
-      // https://github.com/cypress-io/cypress/blob/61156808413be8b99264026323ce3abfb22c4c26/packages/server/lib/modes/results.ts#L20
-      // but it's lost when creating a public test result:
-      // https://github.com/cypress-io/cypress/blob/61156808413be8b99264026323ce3abfb22c4c26/packages/server/lib/modes/results.ts#L111
-      // `.displayError` represents the error reported by the last attempt
-      // for all of the previous attempts we rely on the search for the last errored step in `groupStepsByTest`
-      // if an error is found there, it will be set on the test, the information set here is not overriden though
-      error:
-        attemptIndex === lastAttemptIndex && result.displayError
-          ? {
-              name: "DisplayError",
-              message: result.displayError,
-            }
-          : null,
-    }));
+    return result.attempts.map<Test>((a, attemptIndex) => {
+      const startTestStep = getStartTestStep(result);
+      return {
+        id: startTestStep?.testId ?? attemptIndex,
+        // those properties don't exist since Cypress 13: https://github.com/cypress-io/cypress/pull/27230
+        // TODO: remove it in PRO-640
+        approximateDuration: (a as any).duration || (a as any).wallClockDuration || 0,
+        attempt: attemptIndex + 1,
+        maxAttempts: startTestStep?.maxAttempts ?? 1,
+        source: {
+          title,
+          scope,
+        },
+        result: mapStateToResult(a.state),
+        events: {
+          beforeAll: [],
+          afterAll: [],
+          beforeEach: [],
+          afterEach: [],
+          main: [],
+        },
+        // attempt.error is available here:
+        // https://github.com/cypress-io/cypress/blob/61156808413be8b99264026323ce3abfb22c4c26/packages/server/lib/modes/results.ts#L20
+        // but it's lost when creating a public test result:
+        // https://github.com/cypress-io/cypress/blob/61156808413be8b99264026323ce3abfb22c4c26/packages/server/lib/modes/results.ts#L111
+        // `.displayError` represents the error reported by the last attempt
+        // for all of the previous attempts we rely on the search for the last errored step in `groupStepsByTest`
+        // if an error is found there, it will be set on the test, the information set here is not overriden though
+        error:
+          attemptIndex === lastAttemptIndex && result.displayError
+            ? {
+                name: "DisplayError",
+                message: result.displayError,
+              }
+            : null,
+      };
+    });
   });
 
   debug("Found %d tests", tests.length);

diff --git a/packages/cypress/src/support.ts b/packages/cypress/src/support.ts
@@ -22,6 +22,7 @@ export interface StepEvent {
   timestamp: string;
   testId: number | null;
   attempt: number;
+  maxAttempts?: number | undefined;
   category?: UserActionEvent["data"]["category"];
   hook?: HookKind;
   command?: CommandLike;
@@ -47,6 +48,7 @@ interface CypressMainTestScope {
   scope: string[];
   title: string;
   attempt: number;
+  maxAttempts: number;
   index: number;
   testId: string;
 }
@@ -247,6 +249,7 @@ function getCurrentTestScope(): CypressTestScope {
       title: partialTest.source.title,
       scope: partialTest.source.scope,
       attempt,
+      maxAttempts: (Cypress.getTestRetries() ?? 0) + 1,
       testId: buildTestId(Cypress.spec.relative, partialTest),
       index: order,
     },
@@ -275,6 +278,7 @@ const makeEvent = (
         error,
       }
     : null),
+  ...("maxAttempts" in testScope.v2 ? { maxAttempts: testScope.v2.maxAttempts } : null),
 });
 
 function sendStepToPlugin(event: StepEvent) {

diff --git a/packages/jest/src/runner.ts b/packages/jest/src/runner.ts
@@ -47,7 +47,7 @@ const ReplayRunner = async (
   }
 
   const relativePath = path.relative(config.cwd, testPath);
-  const reporter = new ReplayReporter({ name: "jest", version, plugin: pluginVersion }, "2.1.0");
+  const reporter = new ReplayReporter({ name: "jest", version, plugin: pluginVersion }, "2.2.0");
   reporter.onTestSuiteBegin(undefined, "JEST_REPLAY_METADATA");
 
   function getSource(test: Circus.TestEntry) {
@@ -101,6 +101,7 @@ const ReplayRunner = async (
         {
           id: 0,
           attempt: 1,
+          maxAttempts: 1,
           approximateDuration: test.duration || 0,
           source: getSource(test),
           result: passed ? "passed" : "failed",

diff --git a/packages/playwright/src/reporter.ts b/packages/playwright/src/reporter.ts
@@ -95,7 +95,7 @@ class ReplayPlaywrightReporter implements Reporter {
         version: undefined,
         plugin: pluginVersion,
       },
-      "2.1.0",
+      "2.2.0",
       { ...this.config, metadataKey: "PLAYWRIGHT_REPLAY_METADATA" }
     );
     this.captureTestFile =
@@ -242,16 +242,16 @@ class ReplayPlaywrightReporter implements Reporter {
     // skipped tests won't have a reply so nothing to do here
     if (status === "skipped") return;
 
-    const testMetadata = {
+    const testIdData = {
       id: 0,
       attempt: result.retry + 1,
       source: this.getSource(test),
     };
 
-    const events = this.getStepsFromFixture(testMetadata);
+    const events = this.getStepsFromFixture(testIdData);
 
     const relativePath = test.titlePath()[2];
-    const { stacks, filenames } = this.getFixtureData(testMetadata);
+    const { stacks, filenames } = this.getFixtureData(testIdData);
     filenames.add(test.location.file);
     let playwrightMetadata: Record<string, any> | undefined;
 
@@ -275,7 +275,8 @@ class ReplayPlaywrightReporter implements Reporter {
 
     const tests = [
       {
-        ...testMetadata,
+        ...testIdData,
+        maxAttempts: test.retries + 1,
         approximateDuration: test.results.reduce((acc, r) => acc + r.duration, 0),
         result: status === "interrupted" ? ("unknown" as const) : status,
         error: result.error

diff --git a/packages/replay/src/metadata/test/v2.ts b/packages/replay/src/metadata/test/v2.ts
@@ -117,22 +117,37 @@ const test_v2_1_0 = assign(
   })
 );
 
+const test_v2_2_0 = assign(
+  test_v2_1_0,
+  object({
+    maxAttempts: number(),
+  })
+);
+
 const v2_1_0 = assign(
   v2_0_0,
   object({
     tests: array(test_v2_1_0),
   })
 );
 
+const v2_2_0 = assign(
+  v2_1_0,
+  object({
+    tests: array(test_v2_2_0),
+  })
+);
+
 export namespace TestMetadataV2 {
   export type UserActionEvent = Infer<typeof userActionEvent>;
-  export type Test = Infer<typeof test_v2_1_0>;
+  export type Test = Infer<typeof test_v2_2_0>;
   export type TestResult = Infer<typeof testResult>;
-  export type TestRun = Infer<typeof v2_1_0>;
+  export type TestRun = Infer<typeof v2_2_0>;
   export type TestError = Infer<typeof testError>;
 }
 
 export default {
+  "2.2.0": v2_2_0,
   "2.1.0": v2_1_0,
   "2.0.0": v2_0_0,
 };