From 2c4db3255e15c372f8ae11802f01e49af070535b Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Wed, 28 Feb 2024 14:32:56 -0800 Subject: [PATCH] Split github tokens (#9988) * Split github tokens * Update .ci/gcb-generate-diffs-new.yml Co-authored-by: Stephen Lewis (Burrows) * Remove redundant downstreams token * Make diff processor use new token * Update path to markdown file * Replace GITHUB_TOKEN * Make github tokens optional for generate downstream * Allow either github token to be used * Replace GITHUB_TOKEN * Move environment variable lookup out of constructor * Update .ci/magician/vcr/tester.go Co-authored-by: Stephen Lewis (Burrows) * Add downstream token * Make request reviewer use GITHUB_TOKEN and tgc integration use GITHUB_TOKEN_CLASSIC * Apply suggestions from code review Co-authored-by: Stephen Lewis (Burrows) --------- Co-authored-by: Stephen Lewis (Burrows) --- .ci/gcb-community-checker.yml | 6 +-- .ci/gcb-contributor-membership-checker.yml | 6 +-- .ci/gcb-generate-diffs-new.yml | 36 ++++++++------- .ci/gcb-push-downstream.yml | 44 +++++++++---------- .ci/gcb-vcr-nightly.yml | 2 +- .ci/magician/cmd/check_cassettes.go | 4 +- .ci/magician/cmd/community_checker.go | 7 ++- .ci/magician/cmd/generate_comment.go | 7 +-- .ci/magician/cmd/generate_comment_test.go | 28 ++++++------ .ci/magician/cmd/generate_downstream.go | 20 +++++++-- .ci/magician/cmd/membership_checker.go | 7 ++- .ci/magician/cmd/request_reviewer.go | 7 ++- .ci/magician/cmd/request_service_reviewers.go | 7 ++- .ci/magician/cmd/test_terraform_vcr.go | 7 +-- .ci/magician/cmd/test_tgc.go | 7 ++- .ci/magician/cmd/test_tpg.go | 7 ++- .ci/magician/github/init.go | 15 ++----- .ci/magician/vcr/tester.go | 2 +- .../test_tgc_integration.sh | 6 +-- .../go-plus/vcr-cassette-merger/vcr_merge.sh | 4 +- tools/diff-processor/README.md | 2 +- tools/diff-processor/labels/get_issue.go | 2 +- tools/diff-processor/rules/rule_test.go | 2 +- 23 files changed, 136 insertions(+), 99 deletions(-) diff --git a/.ci/gcb-community-checker.yml b/.ci/gcb-community-checker.yml index 37b2e4955ca1..0eac9a4666e7 100644 --- a/.ci/gcb-community-checker.yml +++ b/.ci/gcb-community-checker.yml @@ -61,7 +61,7 @@ steps: - name: 'gcr.io/graphite-docker-images/go-plus' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' id: community-checker - secretEnv: ["GITHUB_TOKEN", "GENERATE_DIFFS_TRIGGER"] + secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES", "GENERATE_DIFFS_TRIGGER"] timeout: 8000s args: - "community-checker" @@ -74,7 +74,7 @@ steps: availableSecrets: secretManager: - - versionName: projects/673497134629/secrets/github-magician-token/versions/latest - env: GITHUB_TOKEN + - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-magic-modules/versions/latest + env: GITHUB_TOKEN_MAGIC_MODULES - versionName: projects/673497134629/secrets/ci-trigger-generate-diffs/versions/latest env: GENERATE_DIFFS_TRIGGER diff --git a/.ci/gcb-contributor-membership-checker.yml b/.ci/gcb-contributor-membership-checker.yml index c7b2cc3cc89a..f40bc6d69c5b 100644 --- a/.ci/gcb-contributor-membership-checker.yml +++ b/.ci/gcb-contributor-membership-checker.yml @@ -62,7 +62,7 @@ steps: entrypoint: "/workspace/.ci/scripts/go-plus/magician/exec.sh" id: contributor-membership-checker secretEnv: - ["GITHUB_TOKEN", "GENERATE_DIFFS_TRIGGER", "COMMUNITY_CHECKER_TRIGGER"] + ["GITHUB_TOKEN_MAGIC_MODULES", "GENERATE_DIFFS_TRIGGER", "COMMUNITY_CHECKER_TRIGGER"] timeout: 8000s args: - "membership-checker" @@ -75,8 +75,8 @@ steps: availableSecrets: secretManager: - - versionName: projects/673497134629/secrets/github-magician-token/versions/latest - env: GITHUB_TOKEN + - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-magic-modules/versions/latest + env: GITHUB_TOKEN_MAGIC_MODULES - versionName: projects/673497134629/secrets/ci-trigger-generate-diffs/versions/latest env: GENERATE_DIFFS_TRIGGER - versionName: projects/673497134629/secrets/ci-trigger-community-checker/versions/latest diff --git a/.ci/gcb-generate-diffs-new.yml b/.ci/gcb-generate-diffs-new.yml index c19d5c5ebdd9..e775b5a37d39 100644 --- a/.ci/gcb-generate-diffs-new.yml +++ b/.ci/gcb-generate-diffs-new.yml @@ -72,7 +72,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' id: tpg-head - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"] waitFor: ["build-magician-binary"] env: - BASE_BRANCH=$_BASE_BRANCH @@ -86,7 +86,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' id: tpg-base - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"] waitFor: ["build-magician-binary"] env: - BASE_BRANCH=$_BASE_BRANCH @@ -99,7 +99,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"] id: tpgb-head waitFor: ["build-magician-binary"] env: @@ -114,7 +114,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' id: tpgb-base - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"] waitFor: ["build-magician-binary"] env: - BASE_BRANCH=$_BASE_BRANCH @@ -128,7 +128,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' id: tgc-head - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"] waitFor: ["build-magician-binary"] env: - BASE_BRANCH=$_BASE_BRANCH @@ -142,7 +142,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' id: tgc-base - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"] waitFor: ["build-magician-binary"] env: - BASE_BRANCH=$_BASE_BRANCH @@ -156,7 +156,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' id: tf-oics-head - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"] waitFor: ["build-magician-binary"] env: - BASE_BRANCH=$_BASE_BRANCH @@ -170,7 +170,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' id: tf-oics-base - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"] waitFor: ["build-magician-binary"] env: - BASE_BRANCH=$_BASE_BRANCH @@ -184,7 +184,7 @@ steps: - name: 'gcr.io/graphite-docker-images/go-plus' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' id: diff - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES"] args: - 'generate-comment' env: @@ -198,7 +198,7 @@ steps: id: tgc-test allowFailure: true entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES"] waitFor: ["tpgb-head", "tpgb-base", "tgc-head", "tgc-base"] args: - 'test-tgc' @@ -210,7 +210,7 @@ steps: id: tgc-test-integration entrypoint: '/workspace/.ci/scripts/go-plus/tgc-tester-integration/test_tgc_integration.sh' allowFailure: true - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES"] waitFor: ["tpgb-head", "tpgb-base", "tgc-head", "tgc-base"] env: - TEST_PROJECT=$_VALIDATOR_TEST_PROJECT @@ -229,7 +229,7 @@ steps: id: tpgb-test allowFailure: true entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES"] waitFor: ["tpgb-head", "tpgb-base"] args: - 'test-tpg' @@ -242,7 +242,7 @@ steps: id: tpg-test allowFailure: true entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES"] waitFor: ["tpg-head", "tpg-base"] args: - 'test-tpg' @@ -254,7 +254,7 @@ steps: - name: 'gcr.io/graphite-docker-images/go-plus' id: gcb-tpg-vcr-test entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"] waitFor: ["diff"] env: - BASE_BRANCH=$_BASE_BRANCH @@ -271,7 +271,7 @@ steps: - name: 'gcr.io/graphite-docker-images/go-plus' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES"] waitFor: ["diff"] args: - 'request-service-reviewers' @@ -284,8 +284,10 @@ options: availableSecrets: secretManager: - - versionName: projects/673497134629/secrets/github-magician-token/versions/latest - env: GITHUB_TOKEN + - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-downstreams/versions/latest + env: GITHUB_TOKEN_DOWNSTREAMS + - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-magic-modules/versions/latest + env: GITHUB_TOKEN_MAGIC_MODULES - versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest env: GOOGLE_BILLING_ACCOUNT - versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest diff --git a/.ci/gcb-push-downstream.yml b/.ci/gcb-push-downstream.yml index 0f325a15a17c..9b0b58493ab5 100644 --- a/.ci/gcb-push-downstream.yml +++ b/.ci/gcb-push-downstream.yml @@ -33,7 +33,6 @@ steps: - name: 'gcr.io/graphite-docker-images/bash-plus' entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh' id: tpg-sync - secretEnv: ["GITHUB_TOKEN"] waitFor: ["checkout"] args: - 'tpg-sync' @@ -42,7 +41,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_CLASSIC"] id: tpg-push waitFor: ["tpg-sync", "build-magician-binary"] env: @@ -56,22 +55,21 @@ steps: - name: 'gcr.io/cloud-builders/git' waitFor: ["tpg-push"] - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_CLASSIC"] entrypoint: 'bash' args: - -c - | if [ "$BRANCH_NAME" == "main" ]; then - git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpg-sync + git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpg-sync else - git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpg-sync-$BRANCH_NAME + git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpg-sync-$BRANCH_NAME fi # TPGB - name: 'gcr.io/graphite-docker-images/bash-plus' entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh' id: tpgb-sync - secretEnv: ["GITHUB_TOKEN"] waitFor: ["checkout"] args: - 'tpgb-sync' @@ -80,7 +78,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_CLASSIC"] id: tpgb-push waitFor: ["tpgb-sync", "build-magician-binary"] env: @@ -94,22 +92,21 @@ steps: - name: 'gcr.io/cloud-builders/git' waitFor: ["tpgb-push"] - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_CLASSIC"] entrypoint: 'bash' args: - -c - | if [ "$BRANCH_NAME" == "main" ]; then - git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpgb-sync + git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpgb-sync else - git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpgb-sync-$BRANCH_NAME + git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpgb-sync-$BRANCH_NAME fi # TGC - name: 'gcr.io/graphite-docker-images/bash-plus' entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh' id: tgc-sync - secretEnv: ["GITHUB_TOKEN"] waitFor: ["checkout"] args: - 'tgc-sync' @@ -118,7 +115,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_CLASSIC"] id: tgc-push waitFor: ["tgc-sync", "tpgb-push"] env: @@ -132,22 +129,21 @@ steps: - name: 'gcr.io/cloud-builders/git' waitFor: ["tgc-push"] - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_CLASSIC"] entrypoint: 'bash' args: - -c - | if [ "$BRANCH_NAME" == "main" ]; then - git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tgc-sync + git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tgc-sync else - git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tgc-sync-$BRANCH_NAME + git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tgc-sync-$BRANCH_NAME fi # TF-OICS - name: 'gcr.io/graphite-docker-images/bash-plus' entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh' id: tf-oics-sync - secretEnv: ["GITHUB_TOKEN"] waitFor: ["checkout"] args: - 'tf-oics-sync' @@ -156,7 +152,7 @@ steps: - name: 'gcr.io/graphite-docker-images/build-environment' entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_CLASSIC"] id: tf-oics-push waitFor: ["tf-oics-sync", "build-magician-binary"] env: @@ -170,20 +166,20 @@ steps: - name: 'gcr.io/cloud-builders/git' waitFor: ["tf-oics-push"] - secretEnv: ["GITHUB_TOKEN"] + secretEnv: ["GITHUB_TOKEN_CLASSIC"] entrypoint: 'bash' args: - -c - | if [ "$BRANCH_NAME" == "main" ]; then - git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tf-oics-sync + git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tf-oics-sync else - git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tf-oics-sync-$BRANCH_NAME + git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tf-oics-sync-$BRANCH_NAME fi - name: 'gcr.io/graphite-docker-images/go-plus' entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh' - secretEnv: ["GITHUB_TOKEN", "GOOGLE_PROJECT"] + secretEnv: ["GITHUB_TOKEN_CLASSIC", "GOOGLE_PROJECT"] id: vcr-merge waitFor: ["tpg-push"] env: @@ -196,7 +192,7 @@ steps: waitFor: ["vcr-merge"] entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' secretEnv: - - "GITHUB_TOKEN" + - "GITHUB_TOKEN_DOWNSTREAMS" - "GOOGLE_BILLING_ACCOUNT" - "GOOGLE_CUST_ID" - "GOOGLE_FIRESTORE_PROJECT" @@ -228,7 +224,9 @@ logsBucket: 'gs://cloudbuild-downstream-builder-logs' availableSecrets: secretManager: - versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest - env: GITHUB_TOKEN + env: GITHUB_TOKEN_CLASSIC + - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-downstreams/versions/latest + env: GITHUB_TOKEN_DOWNSTREAMS - versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest env: GOOGLE_BILLING_ACCOUNT - versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest diff --git a/.ci/gcb-vcr-nightly.yml b/.ci/gcb-vcr-nightly.yml index fb3bbf8663c3..3664d6312d34 100644 --- a/.ci/gcb-vcr-nightly.yml +++ b/.ci/gcb-vcr-nightly.yml @@ -42,4 +42,4 @@ availableSecrets: - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION - versionName: projects/673497134629/secrets/ci-test-tpu-v2-vm-runtime-version/versions/latest - env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION \ No newline at end of file + env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION diff --git a/.ci/magician/cmd/check_cassettes.go b/.ci/magician/cmd/check_cassettes.go index 321bb83eda69..f5977ec0be29 100644 --- a/.ci/magician/cmd/check_cassettes.go +++ b/.ci/magician/cmd/check_cassettes.go @@ -13,7 +13,7 @@ import ( var ccEnvironmentVariables = [...]string{ "COMMIT_SHA", - "GITHUB_TOKEN", + "GITHUB_TOKEN_DOWNSTREAMS", "GOCACHE", "GOPATH", "GOOGLE_BILLING_ACCOUNT", @@ -62,7 +62,7 @@ var checkCassettesCmd = &cobra.Command{ os.Exit(1) } - ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN"], rnr) + ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN_DOWNSTREAMS"], rnr) vt, err := vcr.NewTester(env, rnr) if err != nil { diff --git a/.ci/magician/cmd/community_checker.go b/.ci/magician/cmd/community_checker.go index f968ca74abff..2ec024952520 100644 --- a/.ci/magician/cmd/community_checker.go +++ b/.ci/magician/cmd/community_checker.go @@ -64,7 +64,12 @@ var communityApprovalCmd = &cobra.Command{ baseBranch := args[5] fmt.Println("Base Branch: ", baseBranch) - gh := github.NewClient() + githubToken, ok := os.LookupEnv("GITHUB_TOKEN_MAGIC_MODULES") + if !ok { + fmt.Println("Did not provide GITHUB_TOKEN_MAGIC_MODULES environment variable") + os.Exit(1) + } + gh := github.NewClient(githubToken) cb := cloudbuild.NewClient() execCommunityChecker(prNumber, commitSha, branchName, headRepoUrl, headBranch, baseBranch, gh, cb) }, diff --git a/.ci/magician/cmd/generate_comment.go b/.ci/magician/cmd/generate_comment.go index f7ad2e5a8b0a..b00ab9986d6e 100644 --- a/.ci/magician/cmd/generate_comment.go +++ b/.ci/magician/cmd/generate_comment.go @@ -35,7 +35,8 @@ var gcEnvironmentVariables = [...]string{ "BUILD_ID", "BUILD_STEP", "COMMIT_SHA", - "GITHUB_TOKEN", + "GITHUB_TOKEN_DOWNSTREAMS", + "GITHUB_TOKEN_MAGIC_MODULES", "GOPATH", "HOME", "PATH", @@ -70,13 +71,13 @@ var generateCommentCmd = &cobra.Command{ env[ev] = val } - gh := github.NewClient() + gh := github.NewClient(env["GITHUB_TOKEN_MAGIC_MODULES"]) rnr, err := exec.NewRunner() if err != nil { fmt.Println("Error creating a runner: ", err) os.Exit(1) } - ctlr := source.NewController(filepath.Join("workspace", "go"), "modular-magician", env["GITHUB_TOKEN"], rnr) + ctlr := source.NewController(filepath.Join("workspace", "go"), "modular-magician", env["GITHUB_TOKEN_DOWNSTREAMS"], rnr) execGenerateComment(env, gh, rnr, ctlr) }, } diff --git a/.ci/magician/cmd/generate_comment_test.go b/.ci/magician/cmd/generate_comment_test.go index 46f90886f180..c8a1c14ed487 100644 --- a/.ci/magician/cmd/generate_comment_test.go +++ b/.ci/magician/cmd/generate_comment_test.go @@ -28,22 +28,22 @@ func TestExecGenerateComment(t *testing.T) { } ctlr := source.NewController("/mock/dir/go", "modular-magician", "*******", mr) env := map[string]string{ - "BUILD_ID": "build1", - "BUILD_STEP": "17", - "COMMIT_SHA": "sha1", - "GITHUB_TOKEN": "*******", - "PR_NUMBER": "pr1", - "PROJECT_ID": "project1", + "BUILD_ID": "build1", + "BUILD_STEP": "17", + "COMMIT_SHA": "sha1", + "GITHUB_TOKEN_MAGIC_MODULES": "*******", + "PR_NUMBER": "pr1", + "PROJECT_ID": "project1", } diffProcessorEnv := map[string]string{ - "BUILD_ID": "build1", - "BUILD_STEP": "17", - "COMMIT_SHA": "sha1", - "GITHUB_TOKEN": "*******", - "NEW_REF": "auto-pr-pr1", - "OLD_REF": "auto-pr-pr1-old", - "PR_NUMBER": "pr1", - "PROJECT_ID": "project1", + "BUILD_ID": "build1", + "BUILD_STEP": "17", + "COMMIT_SHA": "sha1", + "GITHUB_TOKEN_MAGIC_MODULES": "*******", + "NEW_REF": "auto-pr-pr1", + "OLD_REF": "auto-pr-pr1-old", + "PR_NUMBER": "pr1", + "PROJECT_ID": "project1", } execGenerateComment(env, gh, mr, ctlr) diff --git a/.ci/magician/cmd/generate_downstream.go b/.ci/magician/cmd/generate_downstream.go index 033563bda007..f2469d95074a 100644 --- a/.ci/magician/cmd/generate_downstream.go +++ b/.ci/magician/cmd/generate_downstream.go @@ -19,10 +19,14 @@ var changelogExp = regexp.MustCompile("(?s)```release-note.*?```") var gdEnvironmentVariables = [...]string{ "BASE_BRANCH", - "GITHUB_TOKEN", "GOPATH", } +var gdTokenEnvironmentVariables = [...]string{ + "GITHUB_TOKEN_CLASSIC", + "GITHUB_TOKEN_DOWNSTREAMS", +} + var generateDownstreamCmd = &cobra.Command{ Use: "generate-downstream", Short: "Run generate downstream", @@ -47,13 +51,23 @@ var generateDownstreamCmd = &cobra.Command{ env[ev] = val } - gh := github.NewClient() + var githubToken string + for _, ev := range gdTokenEnvironmentVariables { + val, ok := os.LookupEnv(ev) + if ok { + env[ev] = val + githubToken = val + break + } + } + + gh := github.NewClient(githubToken) rnr, err := exec.NewRunner() if err != nil { fmt.Println("Error creating a runner: ", err) os.Exit(1) } - ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN"], rnr) + ctlr := source.NewController(env["GOPATH"], "modular-magician", githubToken, rnr) if len(args) != 4 { fmt.Printf("Wrong number of arguments %d, expected 4\n", len(args)) diff --git a/.ci/magician/cmd/membership_checker.go b/.ci/magician/cmd/membership_checker.go index 4af9c95b81bc..71ffda29a253 100644 --- a/.ci/magician/cmd/membership_checker.go +++ b/.ci/magician/cmd/membership_checker.go @@ -72,7 +72,12 @@ var membershipCheckerCmd = &cobra.Command{ baseBranch := args[5] fmt.Println("Base Branch: ", baseBranch) - gh := github.NewClient() + githubToken, ok := os.LookupEnv("GITHUB_TOKEN_MAGIC_MODULES") + if !ok { + fmt.Println("Did not provide GITHUB_TOKEN_MAGIC_MODULES environment variable") + os.Exit(1) + } + gh := github.NewClient(githubToken) cb := cloudbuild.NewClient() execMembershipChecker(prNumber, commitSha, branchName, headRepoUrl, headBranch, baseBranch, gh, cb) }, diff --git a/.ci/magician/cmd/request_reviewer.go b/.ci/magician/cmd/request_reviewer.go index b5deed4e828e..8ed8661a4e61 100644 --- a/.ci/magician/cmd/request_reviewer.go +++ b/.ci/magician/cmd/request_reviewer.go @@ -48,7 +48,12 @@ var requestReviewerCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { prNumber := args[0] fmt.Println("PR Number: ", prNumber) - gh := github.NewClient() + githubToken, ok := os.LookupEnv("GITHUB_TOKEN") + if !ok { + fmt.Println("Did not provide GITHUB_TOKEN environment variable") + os.Exit(1) + } + gh := github.NewClient(githubToken) execRequestReviewer(prNumber, gh) }, } diff --git a/.ci/magician/cmd/request_service_reviewers.go b/.ci/magician/cmd/request_service_reviewers.go index d1efff417ac5..33abd8d1ca1f 100644 --- a/.ci/magician/cmd/request_service_reviewers.go +++ b/.ci/magician/cmd/request_service_reviewers.go @@ -40,7 +40,12 @@ var requestServiceReviewersCmd = &cobra.Command{ prNumber := args[0] fmt.Println("PR Number: ", prNumber) - gh := github.NewClient() + githubToken, ok := os.LookupEnv("GITHUB_TOKEN_MAGIC_MODULES") + if !ok { + fmt.Println("Did not provide GITHUB_TOKEN_MAGIC_MODULES environment variable") + os.Exit(1) + } + gh := github.NewClient(githubToken) execRequestServiceReviewers(prNumber, gh, labeler.EnrolledTeamsYaml) }, } diff --git a/.ci/magician/cmd/test_terraform_vcr.go b/.ci/magician/cmd/test_terraform_vcr.go index 2749ddd586c0..3c979ef3ab84 100644 --- a/.ci/magician/cmd/test_terraform_vcr.go +++ b/.ci/magician/cmd/test_terraform_vcr.go @@ -15,7 +15,8 @@ import ( ) var ttvEnvironmentVariables = [...]string{ - "GITHUB_TOKEN", + "GITHUB_TOKEN_DOWNSTREAMS", + "GITHUB_TOKEN_MAGIC_MODULES", "GOCACHE", "GOPATH", "GOOGLE_BILLING_ACCOUNT", @@ -59,13 +60,13 @@ var testTerraformVCRCmd = &cobra.Command{ baseBranch = "main" } - gh := github.NewClient() + gh := github.NewClient(env["GITHUB_TOKEN_MAGIC_MODULES"]) rnr, err := exec.NewRunner() if err != nil { fmt.Println("Error creating a runner: ", err) os.Exit(1) } - ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN"], rnr) + ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN_DOWNSTREAMS"], rnr) vt, err := vcr.NewTester(env, rnr) if err != nil { diff --git a/.ci/magician/cmd/test_tgc.go b/.ci/magician/cmd/test_tgc.go index 9f0edf2f29bf..54fd97394612 100644 --- a/.ci/magician/cmd/test_tgc.go +++ b/.ci/magician/cmd/test_tgc.go @@ -36,7 +36,12 @@ var testTGCCmd = &cobra.Command{ commit := os.Getenv("COMMIT_SHA") pr := os.Getenv("PR_NUMBER") - gh := github.NewClient() + githubToken, ok := os.LookupEnv("GITHUB_TOKEN_MAGIC_MODULES") + if !ok { + fmt.Println("Did not provide GITHUB_TOKEN_MAGIC_MODULES environment variable") + os.Exit(1) + } + gh := github.NewClient(githubToken) execTestTGC(commit, pr, gh) }, diff --git a/.ci/magician/cmd/test_tpg.go b/.ci/magician/cmd/test_tpg.go index c8923ad42106..58d216d512ff 100644 --- a/.ci/magician/cmd/test_tpg.go +++ b/.ci/magician/cmd/test_tpg.go @@ -42,7 +42,12 @@ var testTPGCmd = &cobra.Command{ commit := os.Getenv("COMMIT_SHA") pr := os.Getenv("PR_NUMBER") - gh := github.NewClient() + githubToken, ok := os.LookupEnv("GITHUB_TOKEN_MAGIC_MODULES") + if !ok { + fmt.Println("Did not provide GITHUB_TOKEN_MAGIC_MODULES environment variable") + os.Exit(1) + } + gh := github.NewClient(githubToken) execTestTPG(version, commit, pr, gh) }, diff --git a/.ci/magician/github/init.go b/.ci/magician/github/init.go index a931ad5efca5..d64995aa46b7 100644 --- a/.ci/magician/github/init.go +++ b/.ci/magician/github/init.go @@ -15,22 +15,13 @@ */ package github -import ( - "fmt" - "os" -) - // Client for GitHub interactions. type Client struct { token string } -func NewClient() *Client { - githubToken, ok := os.LookupEnv("GITHUB_TOKEN") - if !ok { - fmt.Println("Did not provide GITHUB_TOKEN environment variable") - os.Exit(1) +func NewClient(token string) *Client { + return &Client{ + token: token, } - - return &Client{token: githubToken} } diff --git a/.ci/magician/vcr/tester.go b/.ci/magician/vcr/tester.go index 23e740100916..46e4a36473c1 100644 --- a/.ci/magician/vcr/tester.go +++ b/.ci/magician/vcr/tester.go @@ -199,7 +199,7 @@ func (vt *Tester) Run(mode Mode, version provider.Version, testDirs []string) (* } var printedEnv string for ev, val := range env { - if ev == "SA_KEY" || ev == "GITHUB_TOKEN" { + if ev == "SA_KEY" || strings.HasPrefix(ev, "GITHUB_TOKEN") { val = "{hidden}" } printedEnv += fmt.Sprintf("%s=%s\n", ev, val) diff --git a/.ci/scripts/go-plus/tgc-tester-integration/test_tgc_integration.sh b/.ci/scripts/go-plus/tgc-tester-integration/test_tgc_integration.sh index a17d7099ff44..8f4ead70dc18 100755 --- a/.ci/scripts/go-plus/tgc-tester-integration/test_tgc_integration.sh +++ b/.ci/scripts/go-plus/tgc-tester-integration/test_tgc_integration.sh @@ -12,7 +12,7 @@ github_username=modular-magician new_branch="auto-pr-$pr_number" -git_remote=https://$github_username:$GITHUB_TOKEN@github.com/$github_username/$gh_repo +git_remote=https://github.com/$github_username/$gh_repo local_path=$GOPATH/src/github.com/GoogleCloudPlatform/$gh_repo mkdir -p "$(dirname $local_path)" git clone $git_remote $local_path --branch $new_branch --depth 2 @@ -38,7 +38,7 @@ post_body=$( jq -n \ curl \ -X POST \ - -u "$github_username:$GITHUB_TOKEN" \ + -u "$github_username:$GITHUB_TOKEN_MAGIC_MODULES" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/statuses/$mm_commit_sha" \ -d "$post_body" @@ -68,7 +68,7 @@ post_body=$( jq -n \ curl \ -X POST \ - -u "$github_username:$GITHUB_TOKEN" \ + -u "$github_username:$GITHUB_TOKEN_MAGIC_MODULES" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/statuses/$mm_commit_sha" \ -d "$post_body" diff --git a/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh b/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh index 93d7be0a77a3..7440af55c075 100755 --- a/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh +++ b/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh @@ -11,7 +11,7 @@ else echo "BASE_BRANCH: $BASE_BRANCH" fi -PR_NUMBER=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" \ +PR_NUMBER=$(curl -s -H "Authorization: token ${GITHUB_TOKEN_CLASSIC}" \ "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/pulls?state=closed&base=$BASE_BRANCH&sort=updated&direction=desc" | \ jq -r ".[] | if .merge_commit_sha == \"$REFERENCE\" then .number else empty end") @@ -40,4 +40,4 @@ if [ $? -eq 0 ]; then fi -set -e \ No newline at end of file +set -e diff --git a/tools/diff-processor/README.md b/tools/diff-processor/README.md index 7cd410dd854a..3bb107cae9d1 100644 --- a/tools/diff-processor/README.md +++ b/tools/diff-processor/README.md @@ -16,7 +16,7 @@ bin/diff-processor breaking-changes # Add labels to a PR based on the resources changed between OLD_REF and NEW_REF # The token used must have write access to issues -GITHUB_TOKEN=github_token bin/diff-processor add-labels PR_ID [--dry-run] +GITHUB_TOKEN_MAGIC_MODULES=github_token bin/diff-processor add-labels PR_ID [--dry-run] ``` ## Test diff --git a/tools/diff-processor/labels/get_issue.go b/tools/diff-processor/labels/get_issue.go index fe9b5a235d8a..c9d756892a48 100644 --- a/tools/diff-processor/labels/get_issue.go +++ b/tools/diff-processor/labels/get_issue.go @@ -20,7 +20,7 @@ func GetIssue(repository string, id uint64) (labeler.Issue, error) { return issue, fmt.Errorf("Error creating request: %w", err) } req.Header.Add("Accept", "application/vnd.github+json") - req.Header.Add("Authorization", "Bearer "+os.Getenv("GITHUB_TOKEN")) + req.Header.Add("Authorization", "Bearer "+os.Getenv("GITHUB_TOKEN_MAGIC_MODULES")) req.Header.Add("X-GitHub-Api-Version", "2022-11-28") resp, err := client.Do(req) if err != nil { diff --git a/tools/diff-processor/rules/rule_test.go b/tools/diff-processor/rules/rule_test.go index c68deed47d2c..2ed61fd00c9a 100644 --- a/tools/diff-processor/rules/rule_test.go +++ b/tools/diff-processor/rules/rule_test.go @@ -23,7 +23,7 @@ func TestUniqueRuleIdentifiers(t *testing.T) { func TestMarkdownIdentifiers(t *testing.T) { // Define the Markdown file path relative to the importer - mdFilePath := "../../../docs/content/develop/breaking-changes.md" + mdFilePath := "../../../docs/content/develop/breaking-changes/breaking-changes.md" // Read the Markdown file mdContent, err := ioutil.ReadFile(mdFilePath)