From 63d5041eb4a6f0bbb0db9b12fbd027f7685219ea Mon Sep 17 00:00:00 2001 From: Ryan Duffy Date: Wed, 4 Oct 2023 08:08:48 -0700 Subject: [PATCH] Add user to commit and merge source metadata (#255) --- .github/workflows/ci.yml | 13 +++- Earthfile | 19 ++++- packages/replay/metadata/source.ts | 110 +++++++++++++++++++++++++---- 3 files changed, 126 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdf929d2..2d28abca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,4 +14,15 @@ jobs: - name: Download latest earthly run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.6.19/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'" - name: build, lint and test - run: earthly +ci --REPLAY_API_KEY="${{ secrets.CYPRESS_API_KEY }}" + run: | + cp $GITHUB_EVENT_PATH ./e2e-repos/flake/github_event + earthly +ci \ + --REPLAY_API_KEY="${{ secrets.CYPRESS_API_KEY }}" \ + --GITHUB_SHA="$GITHUB_SHA" \ + --GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \ + --GITHUB_REPOSITORY="$GITHUB_REPOSITORY" \ + --GITHUB_ACTOR="$GITHUB_ACTOR" \ + --GITHUB_RUN_ID="$GITHUB_RUN_ID" \ + --GITHUB_WORKFLOW="$GITHUB_WORKFLOW" \ + --GITHUB_REF_NAME="$GITHUB_REF_NAME" \ + --GITHUB_SERVER_URL="$GITHUB_SERVER_URL" diff --git a/Earthfile b/Earthfile index 441ca0c3..a24bb8ac 100644 --- a/Earthfile +++ b/Earthfile @@ -32,11 +32,28 @@ setup: flake: FROM +setup ARG --required REPLAY_API_KEY + ARG GITHUB_SHA + ARG GITHUB_TOKEN + ARG GITHUB_REPOSITORY + ARG GITHUB_ACTOR + ARG GITHUB_RUN_ID + ARG GITHUB_WORKFLOW + ARG GITHUB_SERVER_URL + ARG GITHUB_REF_NAME WORKDIR /usr/build/e2e-repos/flake ENV REPLAY_METADATA_TEST_RUN_TITLE="flake" ENV REPLAY_API_KEY=${REPLAY_API_KEY} + ENV GITHUB_SHA=${GITHUB_SHA} + ENV GITHUB_TOKEN=${GITHUB_TOKEN} + ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} + ENV GITHUB_ACTOR=${GITHUB_ACTOR} + ENV GITHUB_RUN_ID=${GITHUB_RUN_ID} + ENV GITHUB_WORKFLOW=${GITHUB_WORKFLOW} + ENV GITHUB_SERVER_URL=${GITHUB_SERVER_URL} + ENV GITHUB_REF_NAME=${GITHUB_REF_NAME} + ENV GITHUB_EVENT_PATH=/usr/build/e2e-repos/flake/github_event RUN npm i && npm link @replayio/cypress - RUN DEBUG=replay:*,-replay:cypress:plugin:task,-replay:cypress:plugin:reporter:steps npm run start-and-test || exit 0 + RUN DEBUG=replay:*,-replay:cypress:plugin:task,-replay:cypress:plugin:reporter:steps,replay:cli:metadata:source npm run start-and-test || exit 0 RUN npx @replayio/replay ls --all RUN echo "JUnit Output" RUN find results -type f -exec grep -l 'adding-spec.ts' {} \; | xargs cat diff --git a/packages/replay/metadata/source.ts b/packages/replay/metadata/source.ts index 937af93a..bca5b6aa 100644 --- a/packages/replay/metadata/source.ts +++ b/packages/replay/metadata/source.ts @@ -1,5 +1,6 @@ import { number, Struct } from "superstruct"; -import fetch from "node-fetch"; +import fs from "fs"; +import fetch, { RequestInit, Response } from "node-fetch"; import dbg from "debug"; const { create, object, optional, defaulted } = require("superstruct"); @@ -22,6 +23,33 @@ class GitHubHttpError extends Error { } } +// Add a basic cache so we don't refetch data from GH repeatedly for the same resources +const gFetchCache: Record = {}; +async function fetchWithCache( + url: string, + init?: RequestInit +): Promise<{ json: any | null; status: number; statusText: string }> { + if (!(url in gFetchCache)) { + const resp = await fetch(url, init); + if (resp.status === 200) { + const json = await resp.json(); + gFetchCache[url] = { + status: resp.status, + statusText: resp.statusText, + json, + }; + } else { + gFetchCache[url] = { + json: null, + status: resp.status, + statusText: resp.statusText, + }; + } + } + + return gFetchCache[url]; +} + function getCircleCISourceControlProvider(env: NodeJS.ProcessEnv) { return env.CIRCLE_PULL_REQUEST?.startsWith("https://github.com") ? "github" @@ -37,8 +65,40 @@ function getCircleCIRepository(env: NodeJS.ProcessEnv) { } function getCircleCIMergeId(env: NodeJS.ProcessEnv) { - debug("Extracting merge id from %s", env.CIRCLE_PULL_REQUEST); - return env.CIRCLE_PULL_REQUEST?.split("/").pop(); + if (env.CIRCLE_PULL_REQUEST) { + debug("Extracting merge id from %s", env.CIRCLE_PULL_REQUEST); + return env.CIRCLE_PULL_REQUEST.split("/").pop(); + } +} + +let gGitHubEvent: Record | null = null; +function getGitHubMergeId(env: NodeJS.ProcessEnv) { + const { GITHUB_EVENT_PATH } = env; + if (!GITHUB_EVENT_PATH) { + debug("No github event file specified."); + return; + } + + if (!fs.existsSync(GITHUB_EVENT_PATH)) { + debug("Github event file does not exist at %s", GITHUB_EVENT_PATH); + return; + } + + try { + if (!gGitHubEvent) { + debug("Reading Github event file from %s", GITHUB_EVENT_PATH); + const contents = fs.readFileSync(GITHUB_EVENT_PATH, "utf8"); + gGitHubEvent = JSON.parse(contents); + } else { + debug("Using previously read Github event file"); + } + + if (gGitHubEvent?.pull_request?.number) { + return String(gGitHubEvent.pull_request.number); + } + } catch (e) { + debug("Failed to read pull request number from event: %s", e); + } } async function expandCommitMetadataFromGitHub(repo: string, sha?: string) { @@ -46,6 +106,7 @@ async function expandCommitMetadataFromGitHub(repo: string, sha?: string) { GITHUB_TOKEN, RECORD_REPLAY_METADATA_SOURCE_COMMIT_TITLE, RECORD_REPLAY_METADATA_SOURCE_COMMIT_URL, + RECORD_REPLAY_METADATA_SOURCE_COMMIT_USER, } = process.env; if (!repo || !sha) return; @@ -54,7 +115,7 @@ async function expandCommitMetadataFromGitHub(repo: string, sha?: string) { debug("Fetching commit metadata from %s with %d char token", url, GITHUB_TOKEN?.length || 0); - const resp = await fetch(url, { + const resp = await fetchWithCache(url, { headers: GITHUB_TOKEN ? { Authorization: `token ${GITHUB_TOKEN}`, @@ -63,14 +124,16 @@ async function expandCommitMetadataFromGitHub(repo: string, sha?: string) { }); if (resp.status === 200) { - const json = await resp.json(); + const json = resp.json; process.env.RECORD_REPLAY_METADATA_SOURCE_COMMIT_TITLE = RECORD_REPLAY_METADATA_SOURCE_COMMIT_TITLE || json.commit.message.split("\n").shift().substring(0, 80); process.env.RECORD_REPLAY_METADATA_SOURCE_COMMIT_URL = RECORD_REPLAY_METADATA_SOURCE_COMMIT_URL || json.html_url; + process.env.RECORD_REPLAY_METADATA_SOURCE_COMMIT_USER = + RECORD_REPLAY_METADATA_SOURCE_COMMIT_USER || json.author?.login; } else { - debug("Failed to fetch GitHub commit metadata: %o", resp); + debug("Failed to fetch GitHub commit metadata: %s", resp.statusText); throw new GitHubHttpError(resp.status, resp.statusText); } } @@ -78,17 +141,23 @@ async function expandCommitMetadataFromGitHub(repo: string, sha?: string) { async function expandMergeMetadataFromGitHub(repo: string, pr?: string) { const { GITHUB_TOKEN, + RECORD_REPLAY_METADATA_SOURCE_MERGE_ID, RECORD_REPLAY_METADATA_SOURCE_MERGE_TITLE, RECORD_REPLAY_METADATA_SOURCE_MERGE_URL, + RECORD_REPLAY_METADATA_SOURCE_MERGE_USER, + RECORD_REPLAY_METADATA_SOURCE_BRANCH, } = process.env; - if (!repo || !pr) return; + if (!repo || !pr) { + debug("Unable to retrieve merge metadata: Repo and PR number missing"); + return; + } const url = `https://api.github.com/repos/${repo}/pulls/${pr}`; debug("Fetching merge metadata from %s with %d char token", url, GITHUB_TOKEN?.length || 0); - const resp = await fetch(url, { + const resp = await fetchWithCache(url, { headers: GITHUB_TOKEN ? { Authorization: `token ${GITHUB_TOKEN}`, @@ -97,11 +166,17 @@ async function expandMergeMetadataFromGitHub(repo: string, pr?: string) { }); if (resp.status === 200) { - const json = await resp.json(); + const json = await resp.json; + process.env.RECORD_REPLAY_METADATA_SOURCE_BRANCH = + RECORD_REPLAY_METADATA_SOURCE_BRANCH || json.head?.ref; + process.env.RECORD_REPLAY_METADATA_SOURCE_MERGE_ID = + RECORD_REPLAY_METADATA_SOURCE_MERGE_ID || pr; process.env.RECORD_REPLAY_METADATA_SOURCE_MERGE_TITLE = RECORD_REPLAY_METADATA_SOURCE_MERGE_TITLE || json.title; process.env.RECORD_REPLAY_METADATA_SOURCE_MERGE_URL = RECORD_REPLAY_METADATA_SOURCE_MERGE_URL || json.html_url; + process.env.RECORD_REPLAY_METADATA_SOURCE_MERGE_USER = + RECORD_REPLAY_METADATA_SOURCE_MERGE_USER || json.user?.login; } else { debug("Failed to fetch GitHub commit metadata: %o", resp); throw new GitHubHttpError(resp.status, resp.statusText); @@ -127,6 +202,7 @@ const versions: () => Record = () => ({ ), title: optional(envString("RECORD_REPLAY_METADATA_SOURCE_COMMIT_TITLE")), url: optional(envString("RECORD_REPLAY_METADATA_SOURCE_COMMIT_URL")), + user: optional(envString("RECORD_REPLAY_METADATA_SOURCE_COMMIT_USER")), }), trigger: defaultObject({ user: optional( @@ -153,7 +229,9 @@ const versions: () => Record = () => ({ "RECORD_REPLAY_METADATA_SOURCE_TRIGGER_URL", env => env.GITHUB_WORKFLOW && - `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`, + `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${ + env.GITHUB_REPOSITORY + }/actions/runs/${env.GITHUB_RUN_ID}`, "BUILDKITE_BUILD_URL", "CIRCLE_BUILD_URL" ) @@ -169,6 +247,7 @@ const versions: () => Record = () => ({ ), title: optional(envString("RECORD_REPLAY_METADATA_SOURCE_MERGE_TITLE")), url: optional(envString("RECORD_REPLAY_METADATA_SOURCE_MERGE_URL")), + user: optional(envString("RECORD_REPLAY_METADATA_SOURCE_MERGE_USER")), }), provider: optional( envString( @@ -199,12 +278,15 @@ function validate(metadata: { source: UnstructuredMetadata }) { } async function expandEnvironment() { - const { CIRCLECI, CIRCLE_SHA1 } = process.env; - - const repo = getCircleCIRepository(process.env); + const { CIRCLECI, CIRCLE_SHA1, GITHUB_SHA, GITHUB_REPOSITORY } = process.env; try { - if (CIRCLECI) { + if (GITHUB_SHA && GITHUB_REPOSITORY) { + await expandCommitMetadataFromGitHub(GITHUB_REPOSITORY, GITHUB_SHA); + debug("Merge ID:", getGitHubMergeId(process.env)); + await expandMergeMetadataFromGitHub(GITHUB_REPOSITORY, getGitHubMergeId(process.env)); + } else if (CIRCLECI) { + const repo = getCircleCIRepository(process.env); const provider = getCircleCISourceControlProvider(process.env); if (provider !== "github") {