Skip to content

Commit

Permalink
Adds support for fetching test suite settings from graphql (#223)
Browse files Browse the repository at this point in the history
* Add support for fetching test suite configuration from graphql

* format gql

* wrap fetching config in try/catch to avoid killing tests

* add REPLAY_API_KEY to env before running flake tests

* debug

* less debug

* standardize warning log
ryanjduffy authored Jul 25, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 9155d6a commit f16cc0d
Showing 9 changed files with 103 additions and 11 deletions.
3 changes: 2 additions & 1 deletion Earthfile
Original file line number Diff line number Diff line change
@@ -34,8 +34,9 @@ flake:
ARG --required REPLAY_API_KEY
WORKDIR /usr/build/e2e-repos/flake
ENV REPLAY_METADATA_TEST_RUN_TITLE="flake"
ENV REPLAY_API_KEY=${REPLAY_API_KEY}
RUN npm i && npm link @replayio/cypress
RUN npm run start-and-test || exit 0
RUN DEBUG=replay:*,-replay:cypress:plugin:task,-replay:cypress:plugin:reporter:steps npm run start-and-test || exit 0
DO +UPLOAD --REPLAY_API_KEY=$REPLAY_API_KEY

e2e:
9 changes: 7 additions & 2 deletions packages/cypress/src/mode.ts
Original file line number Diff line number Diff line change
@@ -135,12 +135,17 @@ function getRepeatCount(stressCount = 10) {
}
}

export function getDiagnosticConfig(config: Cypress.PluginConfigOptions): {
export function getDiagnosticConfig(
config: Cypress.PluginConfigOptions,
extraEnv: NodeJS.ProcessEnv = {}
): {
noRecord: boolean;
env: NodeJS.ProcessEnv;
} {
let noRecord = false;
let env: NodeJS.ProcessEnv = {};
let env: NodeJS.ProcessEnv = {
...extraEnv,
};

const { cypress_repeat_k } = config.env;
const repeatIndex = cypress_repeat_k ? Number.parseInt(cypress_repeat_k) : undefined;
9 changes: 5 additions & 4 deletions packages/cypress/src/reporter.ts
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@ import {
ReplayReporter,
TestMetadataV2,
ReporterError,
fetchWorkspaceConfig,
} from "@replayio/test-utils";
import debug from "debug";

@@ -59,12 +60,12 @@ class CypressReporter {

async authenticate(apiKey: string) {
this.reporter.setApiKey(apiKey);
// TODO: we can fetch diagnostics from the cloud later here
this.configureDiagnostics();
const { env } = await fetchWorkspaceConfig(apiKey);
this.configureDiagnostics(env);
}

configureDiagnostics() {
this.diagnosticConfig = getDiagnosticConfig(this.config);
configureDiagnostics(extraEnv?: NodeJS.ProcessEnv) {
this.diagnosticConfig = getDiagnosticConfig(this.config, extraEnv);

// Mix diagnostic env into process env so it can be picked up by test
// metrics and reported to telemetry
76 changes: 76 additions & 0 deletions packages/test-utils/src/config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import fetch from "node-fetch";
import dbg from "debug";
import { warn } from "./logging";

const debug = dbg("replay:test-utils:config");

async function query(apiKey: string, name: string, query: string, variables = {}) {
const options = {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${apiKey}`,
},
body: JSON.stringify({
query,
name,
variables,
}),
};

const server = process.env.REPLAY_API_SERVER || "https://api.replay.io";
debug("Querying %s graphql endpoint", server);
const result = await fetch(`${server}/v1/graphql`, options);

return await result.json();
}

async function fetchWorkspaceConfig(apiKey: string) {
try {
const json = await query(
apiKey,
"GetWorkspaceConfig",
`
query GetWorkspaceConfig {
auth {
workspaces {
edges {
node {
id
settings {
features
}
}
}
}
}
}`
);

if (json.errors) {
debug("GraphQL failed: %s", json.errors);
throw new Error(json.errors[0].message || "Unexpected error");
}

const edges = json.data?.auth.workspaces.edges;
if (!edges || edges.length !== 1) {
debug("Failed to find workspace: %o", json.data);
throw new Error("Failed to find a workspace for the given apiKey");
}

debug("Workspace settings: %o", edges[0].node.settings);
const features = edges[0].node.settings.features;

return {
env: features?.testSuites?.env || {},
};
} catch (e) {
warn("Failed to fetch test suite configuration; continuing with defaults", e);

return {
env: {},
};
}
}

export { fetchWorkspaceConfig };
1 change: 1 addition & 0 deletions packages/test-utils/src/index.ts
Original file line number Diff line number Diff line change
@@ -4,5 +4,6 @@ export type { TestMetadataV1, TestMetadataV2, ReplayReporterConfig } from "./rep
export { ReporterError } from "./reporter";
export { pingTestMetrics } from "./metrics";
export { removeAnsiCodes } from "./terminal";
export { fetchWorkspaceConfig } from "./config";
export { ReplayReporter };
export { getMetadataFilePath, initMetadataFile } from "./metadata";
6 changes: 6 additions & 0 deletions packages/test-utils/src/logging.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
export function warn(message: string, e: unknown) {
console.warn("[replay.io]:", message);
if (e instanceof Error) {
console.warn("[replay.io]: Error:", e.message);
}
}
4 changes: 2 additions & 2 deletions packages/test-utils/src/metadata.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { writeFileSync, existsSync } from "fs";
import path from "path";
import { getDirectory } from "@replayio/replay/src/utils";
import { warn } from "./logging";

export function getMetadataFilePath(base: string, workerIndex = 0) {
return (
@@ -17,8 +18,7 @@ export function initMetadataFile(path: string) {

return path;
} catch (e) {
console.error(`Failed to initialize metadata file${path ? ` at ${path}` : ""}`);
console.error(e);
warn(`Failed to initialize metadata file${path ? ` at ${path}` : ""}`, e);
}

return undefined;
3 changes: 2 additions & 1 deletion packages/test-utils/src/metrics.ts
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ import os from "os";
import fetch from "node-fetch";
import { Test } from "./reporter";
import { TestMetadataV2 } from "@replayio/replay/metadata/test/v2";
import { warn } from "./logging";

const debug = dbg("replay:test-utils:metrics");

@@ -60,7 +61,7 @@ async function pingTestMetrics(
body,
});
} catch (e) {
console.log("Failed to send test metrics", e);
warn("Failed to send test metrics", e);
}
}

3 changes: 2 additions & 1 deletion packages/test-utils/src/reporter.ts
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@ const uuid = require("uuid");

import { getMetadataFilePath } from "./metadata";
import { pingTestMetrics } from "./metrics";
import { warn } from "./logging";

const debug = dbg("replay:test-utils:reporter");

@@ -233,7 +234,7 @@ class ReplayReporter {
mkdirSync(dirname(metadataFilePath), { recursive: true });
writeFileSync(metadataFilePath, JSON.stringify(metadata, undefined, 2), {});
} catch (e) {
console.error("Failed to initialize Replay metadata", e);
warn("Failed to initialize Replay metadata", e);
}
}

0 comments on commit f16cc0d

Please sign in to comment.