From f16cc0dfa1dcf22f3727d163079388f19136a0a1 Mon Sep 17 00:00:00 2001 From: Ryan Duffy Date: Tue, 25 Jul 2023 12:33:50 -0700 Subject: [PATCH] Adds support for fetching test suite settings from graphql (#223) * Add support for fetching test suite configuration from graphql * format gql * wrap fetching config in try/catch to avoid killing tests * add REPLAY_API_KEY to env before running flake tests * debug * less debug * standardize warning log --- Earthfile | 3 +- packages/cypress/src/mode.ts | 9 +++- packages/cypress/src/reporter.ts | 9 ++-- packages/test-utils/src/config.ts | 76 +++++++++++++++++++++++++++++ packages/test-utils/src/index.ts | 1 + packages/test-utils/src/logging.ts | 6 +++ packages/test-utils/src/metadata.ts | 4 +- packages/test-utils/src/metrics.ts | 3 +- packages/test-utils/src/reporter.ts | 3 +- 9 files changed, 103 insertions(+), 11 deletions(-) create mode 100644 packages/test-utils/src/config.ts create mode 100644 packages/test-utils/src/logging.ts diff --git a/Earthfile b/Earthfile index da127507..1dc58f7d 100644 --- a/Earthfile +++ b/Earthfile @@ -34,8 +34,9 @@ flake: ARG --required REPLAY_API_KEY WORKDIR /usr/build/e2e-repos/flake ENV REPLAY_METADATA_TEST_RUN_TITLE="flake" + ENV REPLAY_API_KEY=${REPLAY_API_KEY} RUN npm i && npm link @replayio/cypress - RUN npm run start-and-test || exit 0 + RUN DEBUG=replay:*,-replay:cypress:plugin:task,-replay:cypress:plugin:reporter:steps npm run start-and-test || exit 0 DO +UPLOAD --REPLAY_API_KEY=$REPLAY_API_KEY e2e: diff --git a/packages/cypress/src/mode.ts b/packages/cypress/src/mode.ts index fbcb99e1..79ca71c7 100644 --- a/packages/cypress/src/mode.ts +++ b/packages/cypress/src/mode.ts @@ -135,12 +135,17 @@ function getRepeatCount(stressCount = 10) { } } -export function getDiagnosticConfig(config: Cypress.PluginConfigOptions): { +export function getDiagnosticConfig( + config: Cypress.PluginConfigOptions, + extraEnv: NodeJS.ProcessEnv = {} +): { noRecord: boolean; env: NodeJS.ProcessEnv; } { let noRecord = false; - let env: NodeJS.ProcessEnv = {}; + let env: NodeJS.ProcessEnv = { + ...extraEnv, + }; const { cypress_repeat_k } = config.env; const repeatIndex = cypress_repeat_k ? Number.parseInt(cypress_repeat_k) : undefined; diff --git a/packages/cypress/src/reporter.ts b/packages/cypress/src/reporter.ts index 1675ab2a..1293d284 100644 --- a/packages/cypress/src/reporter.ts +++ b/packages/cypress/src/reporter.ts @@ -4,6 +4,7 @@ import { ReplayReporter, TestMetadataV2, ReporterError, + fetchWorkspaceConfig, } from "@replayio/test-utils"; import debug from "debug"; @@ -59,12 +60,12 @@ class CypressReporter { async authenticate(apiKey: string) { this.reporter.setApiKey(apiKey); - // TODO: we can fetch diagnostics from the cloud later here - this.configureDiagnostics(); + const { env } = await fetchWorkspaceConfig(apiKey); + this.configureDiagnostics(env); } - configureDiagnostics() { - this.diagnosticConfig = getDiagnosticConfig(this.config); + configureDiagnostics(extraEnv?: NodeJS.ProcessEnv) { + this.diagnosticConfig = getDiagnosticConfig(this.config, extraEnv); // Mix diagnostic env into process env so it can be picked up by test // metrics and reported to telemetry diff --git a/packages/test-utils/src/config.ts b/packages/test-utils/src/config.ts new file mode 100644 index 00000000..27fd11a6 --- /dev/null +++ b/packages/test-utils/src/config.ts @@ -0,0 +1,76 @@ +import fetch from "node-fetch"; +import dbg from "debug"; +import { warn } from "./logging"; + +const debug = dbg("replay:test-utils:config"); + +async function query(apiKey: string, name: string, query: string, variables = {}) { + const options = { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + query, + name, + variables, + }), + }; + + const server = process.env.REPLAY_API_SERVER || "https://api.replay.io"; + debug("Querying %s graphql endpoint", server); + const result = await fetch(`${server}/v1/graphql`, options); + + return await result.json(); +} + +async function fetchWorkspaceConfig(apiKey: string) { + try { + const json = await query( + apiKey, + "GetWorkspaceConfig", + ` + query GetWorkspaceConfig { + auth { + workspaces { + edges { + node { + id + settings { + features + } + } + } + } + } + }` + ); + + if (json.errors) { + debug("GraphQL failed: %s", json.errors); + throw new Error(json.errors[0].message || "Unexpected error"); + } + + const edges = json.data?.auth.workspaces.edges; + if (!edges || edges.length !== 1) { + debug("Failed to find workspace: %o", json.data); + throw new Error("Failed to find a workspace for the given apiKey"); + } + + debug("Workspace settings: %o", edges[0].node.settings); + const features = edges[0].node.settings.features; + + return { + env: features?.testSuites?.env || {}, + }; + } catch (e) { + warn("Failed to fetch test suite configuration; continuing with defaults", e); + + return { + env: {}, + }; + } +} + +export { fetchWorkspaceConfig }; diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts index 7a09d574..9d2f6cad 100644 --- a/packages/test-utils/src/index.ts +++ b/packages/test-utils/src/index.ts @@ -4,5 +4,6 @@ export type { TestMetadataV1, TestMetadataV2, ReplayReporterConfig } from "./rep export { ReporterError } from "./reporter"; export { pingTestMetrics } from "./metrics"; export { removeAnsiCodes } from "./terminal"; +export { fetchWorkspaceConfig } from "./config"; export { ReplayReporter }; export { getMetadataFilePath, initMetadataFile } from "./metadata"; diff --git a/packages/test-utils/src/logging.ts b/packages/test-utils/src/logging.ts new file mode 100644 index 00000000..b22d4d74 --- /dev/null +++ b/packages/test-utils/src/logging.ts @@ -0,0 +1,6 @@ +export function warn(message: string, e: unknown) { + console.warn("[replay.io]:", message); + if (e instanceof Error) { + console.warn("[replay.io]: Error:", e.message); + } +} diff --git a/packages/test-utils/src/metadata.ts b/packages/test-utils/src/metadata.ts index 8019cf0d..1dafb8da 100644 --- a/packages/test-utils/src/metadata.ts +++ b/packages/test-utils/src/metadata.ts @@ -1,6 +1,7 @@ import { writeFileSync, existsSync } from "fs"; import path from "path"; import { getDirectory } from "@replayio/replay/src/utils"; +import { warn } from "./logging"; export function getMetadataFilePath(base: string, workerIndex = 0) { return ( @@ -17,8 +18,7 @@ export function initMetadataFile(path: string) { return path; } catch (e) { - console.error(`Failed to initialize metadata file${path ? ` at ${path}` : ""}`); - console.error(e); + warn(`Failed to initialize metadata file${path ? ` at ${path}` : ""}`, e); } return undefined; diff --git a/packages/test-utils/src/metrics.ts b/packages/test-utils/src/metrics.ts index d67a88d7..24d3b856 100644 --- a/packages/test-utils/src/metrics.ts +++ b/packages/test-utils/src/metrics.ts @@ -3,6 +3,7 @@ import os from "os"; import fetch from "node-fetch"; import { Test } from "./reporter"; import { TestMetadataV2 } from "@replayio/replay/metadata/test/v2"; +import { warn } from "./logging"; const debug = dbg("replay:test-utils:metrics"); @@ -60,7 +61,7 @@ async function pingTestMetrics( body, }); } catch (e) { - console.log("Failed to send test metrics", e); + warn("Failed to send test metrics", e); } } diff --git a/packages/test-utils/src/reporter.ts b/packages/test-utils/src/reporter.ts index d7b318bf..cac38b53 100644 --- a/packages/test-utils/src/reporter.ts +++ b/packages/test-utils/src/reporter.ts @@ -8,6 +8,7 @@ const uuid = require("uuid"); import { getMetadataFilePath } from "./metadata"; import { pingTestMetrics } from "./metrics"; +import { warn } from "./logging"; const debug = dbg("replay:test-utils:reporter"); @@ -233,7 +234,7 @@ class ReplayReporter { mkdirSync(dirname(metadataFilePath), { recursive: true }); writeFileSync(metadataFilePath, JSON.stringify(metadata, undefined, 2), {}); } catch (e) { - console.error("Failed to initialize Replay metadata", e); + warn("Failed to initialize Replay metadata", e); } }