Add Mixpanel to Playwright (#577)

- [x] Reorganized Mixpanel code to be a bit easier to reason about and added some more unit tests
  - Most of the lines changed in this PR comes from this reorganization
- [x] Add some basic Mixpanel logging to the `playwright` and `test-utils` packages
  - Playwright specific tracking:
    - Invalid reporter config
    - Replay reporter used without Replay browser
  - Test utils tracking (also applies to Cypress)
    - Reporter error
    - Missing API key
    - Recording metadata
    - Test end
    - Test suite begin and end
    - Test suite results (including number of crashed recordings and uploaded recordings)
- [x] Spot test and verify that events are ending up in Mixpanel

packages/cypress/package.json

@@ -52,6 +52,7 @@
     "is-uuid": "^1.0.2",
     "jsonata": "^1.8.6",
     "launchdarkly-node-client-sdk": "^3.2.1",
+    "mixpanel": "^0.18.0",
     "node-fetch": "^2.6.7",
     "p-map": "^4.0.0",
     "semver": "^7.5.2",

packages/jest/package.json

@@ -55,6 +55,7 @@
     "jest-runtime": "^27.5.1",
     "jsonata": "^1.8.6",
     "launchdarkly-node-client-sdk": "^3.2.1",
+    "mixpanel": "^0.18.0",
     "node-fetch": "^2.6.7",
     "p-map": "^4.0.0",
     "sha-1": "^1.0.0",

packages/playwright/package.json

@@ -50,6 +50,7 @@
     "is-uuid": "^1.0.2",
     "jsonata": "^1.8.6",
     "launchdarkly-node-client-sdk": "^3.2.1",
+    "mixpanel": "^0.18.0",
     "node-fetch": "^2.6.7",
     "p-map": "^4.0.0",
     "sha-1": "^1.0.0",

packages/playwright/src/reporter.ts

@@ -5,28 +5,29 @@ import type {
   TestError,
   TestResult,
 } from "@playwright/test/reporter";
+import { initLogger, logger } from "@replay-cli/shared/logger";
+import { mixpanelAPI } from "@replay-cli/shared/mixpanel/mixpanelAPI";
+import { getRuntimePath } from "@replay-cli/shared/runtime/getRuntimePath";
 import { emphasize, highlight, link } from "@replay-cli/shared/theme";
+import { setUserAgent } from "@replay-cli/shared/userAgent";
 import {
   ReplayReporter,
   ReplayReporterConfig,
   TestMetadataV2,
+  getAccessToken,
   getMetadataFilePath as getMetadataFilePathBase,
   removeAnsiCodes,
 } from "@replayio/test-utils";
 import { existsSync, readFileSync } from "fs";
 import path from "path";
 import { WebSocketServer } from "ws";
-
-type UserActionEvent = TestMetadataV2.UserActionEvent;
-
-import { initLogger, logger } from "@replay-cli/shared/logger";
-import { getRuntimePath } from "@replay-cli/shared/runtime/getRuntimePath";
-import { setUserAgent } from "@replay-cli/shared/userAgent";
-import pkgJson from "../package.json";
+import { name as packageName, version as packageVersion } from "../package.json";
 import { FixtureStepStart, ParsedErrorFrame, TestExecutionIdData } from "./fixture";
 import { StackFrame } from "./playwrightTypes";
 import { getServerPort, startServer } from "./server";
 
+type UserActionEvent = TestMetadataV2.UserActionEvent;
+
 export function getMetadataFilePath(workerIndex = 0) {
   return getMetadataFilePathBase("PLAYWRIGHT", workerIndex);
 }
@@ -69,7 +70,7 @@ interface FixtureStep extends FixtureStepStart {
   error?: ParsedErrorFrame | undefined;
 }
 
-class ReplayPlaywrightReporter implements Reporter {
+export default class ReplayPlaywrightReporter implements Reporter {
   reporter: ReplayReporter<ReplayPlaywrightRecordingMetadata>;
   captureTestFile: boolean;
   config: ReplayPlaywrightConfig;
@@ -81,9 +82,18 @@ class ReplayPlaywrightReporter implements Reporter {
   private _foundReplayBrowser = false;
 
   constructor(config: ReplayPlaywrightConfig) {
-    setUserAgent(`${pkgJson.name}/${pkgJson.version}`);
-    initLogger(pkgJson.name, pkgJson.version);
+    setUserAgent(`${packageName}/${packageVersion}`);
+
+    initLogger(packageName, packageVersion);
+    mixpanelAPI.initialize({
+      accessToken: getAccessToken(config),
+      packageName,
+      packageVersion,
+    });
+
     if (!config || typeof config !== "object") {
+      mixpanelAPI.trackEvent("playwright.error.invalid-reporter-config", { config });
+
       throw new Error(
         `Expected an object for @replayio/playwright/reporter configuration but received: ${config}`
       );
@@ -94,9 +104,9 @@ class ReplayPlaywrightReporter implements Reporter {
       {
         name: "playwright",
         version: undefined,
-        plugin: pkgJson.version,
+        plugin: packageVersion,
       },
-      "2.2.0",
+      "2.2.0", // Schema version
       { ...this.config, metadataKey: "PLAYWRIGHT_REPLAY_METADATA" }
     );
     this.captureTestFile =
@@ -123,8 +133,10 @@ class ReplayPlaywrightReporter implements Reporter {
           s.error = step.error;
         }
       },
-      onError: (_test, error) => {
-        this.reporter?.addError(error);
+      onError: (test, error) => {
+        this.reporter?.addError(error, {
+          ...test,
+        });
       },
     });
   }
@@ -329,6 +341,8 @@ class ReplayPlaywrightReporter implements Reporter {
     try {
       await this.reporter.onEnd();
       if (!this._foundReplayBrowser) {
+        mixpanelAPI.trackEvent("playwright.warning.reporter-used-without-browser");
+
         const output: string[] = [];
         output.push(emphasize("None of the configured projects ran using Replay Chromium."));
         if (!existsSync(getRuntimePath())) {
@@ -344,7 +358,7 @@ class ReplayPlaywrightReporter implements Reporter {
         output.map(line => console.warn(`[replay.io]: ${line}`));
       }
     } finally {
-      await logger.close().catch(() => {});
+      await Promise.all([mixpanelAPI.close().catch(noop), logger.close().catch(noop)]);
     }
   }
 
@@ -385,4 +399,4 @@ class ReplayPlaywrightReporter implements Reporter {
   }
 }
 
-export default ReplayPlaywrightReporter;
+function noop() {}

packages/puppeteer/package.json

@@ -43,6 +43,7 @@
     "is-uuid": "^1.0.2",
     "jsonata": "^1.8.6",
     "launchdarkly-node-client-sdk": "^3.2.1",
+    "mixpanel": "^0.18.0",
     "node-fetch": "^2.6.7",
     "p-map": "^4.0.0",
     "sha-1": "^1.0.0",

packages/replayio/src/commands/record.ts

@@ -1,5 +1,5 @@
 import { ProcessError } from "@replay-cli/shared/ProcessError";
-import { trackEvent } from "@replay-cli/shared/mixpanel/trackEvent";
+import { mixpanelAPI } from "@replay-cli/shared/mixpanel/mixpanelAPI";
 import { exitProcess } from "@replay-cli/shared/process/exitProcess";
 import { canUpload } from "@replay-cli/shared/recording/canUpload";
 import { getRecordings } from "@replay-cli/shared/recording/getRecordings";
@@ -101,7 +101,7 @@ async function record(url: string = "about:blank") {
     console.log(""); // Spacing for readability
   }
 
-  trackEvent("record.results", {
+  mixpanelAPI.trackEvent("record.results", {
     crashedCount: crashedRecordings.length,
     successCountsByType: finishedRecordings.reduce(
       (map, recording) => {

packages/replayio/src/utils/commander/registerCommand.ts

@@ -1,4 +1,4 @@
-import { trackEvent } from "@replay-cli/shared/mixpanel/trackEvent";
+import { mixpanelAPI } from "@replay-cli/shared/mixpanel/mixpanelAPI";
 import { program } from "commander";
 import { initialize } from "../initialization/initialize";
 
@@ -17,7 +17,7 @@ export function registerCommand(
   } = config;
 
   return program.command(commandName).hook("preAction", async () => {
-    trackEvent("command", { commandName });
+    mixpanelAPI.trackEvent("command", { commandName });
 
     await initialize({
       checkForNpmUpdate,

packages/replayio/src/utils/initialization/checkForNpmUpdate.ts

@@ -1,13 +1,13 @@
 import { logger } from "@replay-cli/shared/logger";
-import { withTrackAsyncEvent } from "@replay-cli/shared/mixpanel/withTrackAsyncEvent";
+import { createAsyncFunctionWithTracking } from "@replay-cli/shared/mixpanel/createAsyncFunctionWithTracking";
 import { fetch } from "undici";
 import { version as currentVersion, name as packageName } from "../../../package.json";
 import { shouldPrompt } from "../prompt/shouldPrompt";
 import { UpdateCheck } from "./types";
 
 const PROMPT_ID = "npm-update";
 
-export const checkForNpmUpdate = withTrackAsyncEvent(
+export const checkForNpmUpdate = createAsyncFunctionWithTracking(
   async function checkForNpmUpdate(): Promise<UpdateCheck<string>> {
     try {
       // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-metadata-format

packages/replayio/src/utils/initialization/checkForRuntimeUpdate.ts

@@ -1,5 +1,5 @@
 import { logger } from "@replay-cli/shared/logger";
-import { withTrackAsyncEvent } from "@replay-cli/shared/mixpanel/withTrackAsyncEvent";
+import { createAsyncFunctionWithTracking } from "@replay-cli/shared/mixpanel/createAsyncFunctionWithTracking";
 import { existsSync } from "fs-extra";
 import { getBrowserPath } from "../browser/getBrowserPath";
 import { getLatestRelease } from "../installation/getLatestReleases";
@@ -15,7 +15,7 @@ export type Version = {
   version: Release["version"];
 };
 
-export const checkForRuntimeUpdate = withTrackAsyncEvent(
+export const checkForRuntimeUpdate = createAsyncFunctionWithTracking(
   async function checkForRuntimeUpdate(): Promise<UpdateCheck<Version>> {
     let latestRelease: Release;
     let latestBuildId: string;

packages/replayio/src/utils/initialization/initialize.ts

@@ -1,7 +1,7 @@
 import { raceWithTimeout } from "@replay-cli/shared/async/raceWithTimeout";
 import { getAccessToken } from "@replay-cli/shared/authentication/getAccessToken";
 import { initLaunchDarklyFromAccessToken } from "@replay-cli/shared/launch-darkly/initLaunchDarklyFromAccessToken";
-import { initMixpanelForUserSession } from "@replay-cli/shared/mixpanel/initMixpanelForUserSession";
+import { mixpanelAPI } from "@replay-cli/shared/mixpanel/mixpanelAPI";
 import { name as packageName, version as packageVersion } from "../../../package.json";
 import { logPromise } from "../async/logPromise";
 import { checkForNpmUpdate } from "./checkForNpmUpdate";
@@ -60,10 +60,7 @@ export async function initialize({
     : Promise.resolve();
 
   const mixpanelPromise = raceWithTimeout(
-    initMixpanelForUserSession(accessToken, {
-      packageName,
-      packageVersion,
-    }),
+    mixpanelAPI.initialize({ accessToken, packageName, packageVersion }),
     2_500,
     abortController
   );

packages/replayio/src/utils/initialization/promptForRuntimeUpdate.ts

@@ -1,4 +1,4 @@
-import { trackEvent } from "@replay-cli/shared/mixpanel/trackEvent";
+import { mixpanelAPI } from "@replay-cli/shared/mixpanel/mixpanelAPI";
 import { emphasize } from "@replay-cli/shared/theme";
 import { name as packageName } from "../../../package.json";
 import { installLatestRelease } from "../installation/installLatestRelease";
@@ -50,7 +50,7 @@ export async function promptForRuntimeUpdate(updateCheck: UpdateCheckResult<Vers
       // A failed install will be handled later
     }
   } else {
-    trackEvent("update.runtime.skipped", { newRuntimeVersion: toVersion });
+    mixpanelAPI.trackEvent("update.runtime.skipped", { newRuntimeVersion: toVersion });
   }
 
   console.log("");

packages/replayio/src/utils/installation/installLatestRelease.ts

@@ -3,7 +3,7 @@
 import { writeToCache } from "@replay-cli/shared/cache";
 import { getReplayPath } from "@replay-cli/shared/getReplayPath";
 import { logger } from "@replay-cli/shared/logger";
-import { withTrackAsyncEvent } from "@replay-cli/shared/mixpanel/withTrackAsyncEvent";
+import { createAsyncFunctionWithTracking } from "@replay-cli/shared/mixpanel/createAsyncFunctionWithTracking";
 import { dim, link } from "@replay-cli/shared/theme";
 import { spawnSync } from "child_process";
 import { ensureDirSync, renameSync, rmSync, unlinkSync, writeFileSync } from "fs-extra";
@@ -21,7 +21,7 @@ type Result = {
   forkedVersion: string | null;
 };
 
-export const installLatestRelease = withTrackAsyncEvent(
+export const installLatestRelease = createAsyncFunctionWithTracking(
   async function installLatestRelease(): Promise<Result | undefined> {
     const runtimeBaseDir = getReplayPath("runtimes");
     const runtimePath = getReplayPath("runtimes", runtimeMetadata.destinationName);

packages/shared/src/mixpanel/withTrackAsyncEvent.test.ts → packages/shared/src/mixpanel/createAsyncFunctionWithTracking.test.ts

@@ -1,24 +1,33 @@
 import { Deferred, createDeferred } from "../async/createDeferred";
-import { MixpanelAPI } from "./types";
+import type { mixpanelAPI as MixpanelAPIType, MixpanelImplementation } from "./mixpanelAPI";
 
-describe("withTrackAsyncEvent", () => {
-  let mockMixpanelAPI: MixpanelAPI;
-  let withTrackAsyncEvent: typeof import("./withTrackAsyncEvent").withTrackAsyncEvent;
+describe("createAsyncFunctionWithTracking", () => {
+  let createAsyncFunctionWithTracking: typeof import("./createAsyncFunctionWithTracking").createAsyncFunctionWithTracking;
+  let mockMixpanelAPI: MixpanelImplementation;
+  let mixpanelAPI: typeof MixpanelAPIType;
 
   beforeEach(() => {
     mockMixpanelAPI = {
       init: jest.fn(),
       track: jest.fn(),
     };
 
-    // jest.resetModules does not work with import; only works with require()
-    withTrackAsyncEvent = require("./withTrackAsyncEvent").withTrackAsyncEvent;
+    jest.mock("../graphql/getAuthInfo", () => ({
+      getAuthInfo: async () => ({
+        id: "fake-session-id",
+      }),
+    }));
 
-    require("./session").configureSession("fake-user-id", {
-      packageName: "fake-name",
+    // jest.resetModules does not work with import; only works with require()
+    createAsyncFunctionWithTracking =
+      require("./createAsyncFunctionWithTracking").createAsyncFunctionWithTracking;
+    mixpanelAPI = require("./mixpanelAPI").mixpanelAPI;
+    mixpanelAPI.mockForTests(mockMixpanelAPI);
+    mixpanelAPI.initialize({
+      accessToken: "fake-access-token",
+      packageName: "fake-package",
       packageVersion: "0.0.0",
     });
-    require("./getMixpanelAPI").setMixpanelAPIForTests(mockMixpanelAPI);
   });
 
   afterEach(() => {
@@ -34,7 +43,7 @@ describe("withTrackAsyncEvent", () => {
       anotherProperty: "another",
     }));
 
-    const callbackWithTracking = withTrackAsyncEvent(
+    const callbackWithTracking = createAsyncFunctionWithTracking(
       () => deferred.promise,
       "test-event",
       mockGetProperties
@@ -54,7 +63,7 @@ describe("withTrackAsyncEvent", () => {
     const properties = (mockMixpanelAPI.track as jest.Mock).mock.calls[0][1];
     expect(properties).toMatchObject({
       anotherProperty: "another",
-      distinct_id: "fake-user-id",
+      distinct_id: "fake-session-id",
       error: undefined,
       result: "result",
     });
@@ -66,7 +75,7 @@ describe("withTrackAsyncEvent", () => {
       result,
     }));
 
-    const callbackWithTracking = withTrackAsyncEvent(
+    const callbackWithTracking = createAsyncFunctionWithTracking(
       (foo, bar) => Promise.resolve({ foo, bar }),
       "test-event",
       mockGetProperties
@@ -80,7 +89,7 @@ describe("withTrackAsyncEvent", () => {
 
     const properties = (mockMixpanelAPI.track as jest.Mock).mock.calls[0][1];
     expect(properties).toMatchObject({
-      distinct_id: "fake-user-id",
+      distinct_id: "fake-session-id",
       error: undefined,
       result: { foo: "abc", bar: 123 },
     });
@@ -93,7 +102,7 @@ describe("withTrackAsyncEvent", () => {
       result,
     }));
 
-    const callbackWithTracking = withTrackAsyncEvent(
+    const callbackWithTracking = createAsyncFunctionWithTracking(
       () => {
         const deferred = createDeferred<string>();
         deferredArray.push(deferred);

packages/shared/src/mixpanel/createAsyncFunctionWithTracking.ts

@@ -0,0 +1,10 @@
+import { mixpanelAPI, Properties } from "./mixpanelAPI";
+
+export function createAsyncFunctionWithTracking<Params extends Array<any>, Type>(
+  createPromise: (...args: Params) => Promise<Type>,
+  eventName: string,
+  properties?: Properties | ((result: Type | undefined, error: any) => Properties)
+): (...args: Params) => Promise<Type> {
+  return (...args: Params) =>
+    mixpanelAPI.trackAsyncEvent(createPromise(...args), eventName, properties);
+}