Split github tokens (#9988)

* Split github tokens * Update .ci/gcb-generate-diffs-new.yml Co-authored-by: Stephen Lewis (Burrows) <[email protected]> * Remove redundant downstreams token * Make diff processor use new token * Update path to markdown file * Replace GITHUB_TOKEN * Make github tokens optional for generate downstream * Allow either github token to be used * Replace GITHUB_TOKEN * Move environment variable lookup out of constructor * Update .ci/magician/vcr/tester.go Co-authored-by: Stephen Lewis (Burrows) <[email protected]> * Add downstream token * Make request reviewer use GITHUB_TOKEN and tgc integration use GITHUB_TOKEN_CLASSIC * Apply suggestions from code review Co-authored-by: Stephen Lewis (Burrows) <[email protected]> --------- Co-authored-by: Stephen Lewis (Burrows) <[email protected]>
GoogleCloudPlatform · Feb 28, 2024 · 2c4db32 · 2c4db32
1 parent 92183ab
commit 2c4db32
Show file tree

Hide file tree

Showing 23 changed files with 136 additions and 99 deletions.
diff --git a/.ci/gcb-community-checker.yml b/.ci/gcb-community-checker.yml
@@ -61,7 +61,7 @@ steps:
   - name: 'gcr.io/graphite-docker-images/go-plus'
     entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
     id: community-checker
-    secretEnv: ["GITHUB_TOKEN", "GENERATE_DIFFS_TRIGGER"]
+    secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES", "GENERATE_DIFFS_TRIGGER"]
     timeout: 8000s
     args:
         - "community-checker"
@@ -74,7 +74,7 @@ steps:
 
 availableSecrets:
   secretManager:
-    - versionName: projects/673497134629/secrets/github-magician-token/versions/latest
-      env: GITHUB_TOKEN
+    - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-magic-modules/versions/latest
+      env: GITHUB_TOKEN_MAGIC_MODULES
     - versionName: projects/673497134629/secrets/ci-trigger-generate-diffs/versions/latest
       env: GENERATE_DIFFS_TRIGGER
diff --git a/.ci/gcb-contributor-membership-checker.yml b/.ci/gcb-contributor-membership-checker.yml
@@ -62,7 +62,7 @@ steps:
     entrypoint: "/workspace/.ci/scripts/go-plus/magician/exec.sh"
     id: contributor-membership-checker
     secretEnv:
-      ["GITHUB_TOKEN", "GENERATE_DIFFS_TRIGGER", "COMMUNITY_CHECKER_TRIGGER"]
+      ["GITHUB_TOKEN_MAGIC_MODULES", "GENERATE_DIFFS_TRIGGER", "COMMUNITY_CHECKER_TRIGGER"]
     timeout: 8000s
     args:
       - "membership-checker"
@@ -75,8 +75,8 @@ steps:
 
 availableSecrets:
   secretManager:
-    - versionName: projects/673497134629/secrets/github-magician-token/versions/latest
-      env: GITHUB_TOKEN
+    - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-magic-modules/versions/latest
+      env: GITHUB_TOKEN_MAGIC_MODULES
     - versionName: projects/673497134629/secrets/ci-trigger-generate-diffs/versions/latest
       env: GENERATE_DIFFS_TRIGGER
     - versionName: projects/673497134629/secrets/ci-trigger-community-checker/versions/latest

diff --git a/.ci/gcb-generate-diffs-new.yml b/.ci/gcb-generate-diffs-new.yml
@@ -72,7 +72,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tpg-head
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"]
       waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
@@ -86,7 +86,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tpg-base
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"]
       waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
@@ -99,7 +99,7 @@ steps:
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"]
       id: tpgb-head
       waitFor: ["build-magician-binary"]
       env:
@@ -114,7 +114,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tpgb-base
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"]
       waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
@@ -128,7 +128,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tgc-head
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"]
       waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
@@ -142,7 +142,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tgc-base
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"]
       waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
@@ -156,7 +156,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tf-oics-head
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"]
       waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
@@ -170,7 +170,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tf-oics-base
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS"]
       waitFor: ["build-magician-binary"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
@@ -184,7 +184,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: diff
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES"]
       args:
           - 'generate-comment'
       env:
@@ -198,7 +198,7 @@ steps:
       id: tgc-test
       allowFailure: true
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES"]
       waitFor: ["tpgb-head", "tpgb-base", "tgc-head", "tgc-base"]
       args:
         - 'test-tgc'
@@ -210,7 +210,7 @@ steps:
       id: tgc-test-integration
       entrypoint: '/workspace/.ci/scripts/go-plus/tgc-tester-integration/test_tgc_integration.sh'
       allowFailure: true
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES"]
       waitFor: ["tpgb-head", "tpgb-base", "tgc-head", "tgc-base"]
       env:
         - TEST_PROJECT=$_VALIDATOR_TEST_PROJECT
@@ -229,7 +229,7 @@ steps:
       id: tpgb-test
       allowFailure: true
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES"]
       waitFor: ["tpgb-head", "tpgb-base"]
       args:
         - 'test-tpg'
@@ -242,7 +242,7 @@ steps:
       id: tpg-test
       allowFailure: true
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES"]
       waitFor: ["tpg-head", "tpg-base"]
       args:
         - 'test-tpg'
@@ -254,7 +254,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: gcb-tpg-vcr-test
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"]
       waitFor: ["diff"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
@@ -271,7 +271,7 @@ steps:
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES"]
       waitFor: ["diff"]
       args:
           - 'request-service-reviewers'
@@ -284,8 +284,10 @@ options:
 
 availableSecrets:
   secretManager:
-    - versionName: projects/673497134629/secrets/github-magician-token/versions/latest
-      env: GITHUB_TOKEN
+    - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-downstreams/versions/latest
+      env: GITHUB_TOKEN_DOWNSTREAMS
+    - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-magic-modules/versions/latest
+      env: GITHUB_TOKEN_MAGIC_MODULES
     - versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest
       env: GOOGLE_BILLING_ACCOUNT
     - versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest

diff --git a/.ci/gcb-push-downstream.yml b/.ci/gcb-push-downstream.yml
@@ -33,7 +33,6 @@ steps:
     - name: 'gcr.io/graphite-docker-images/bash-plus'
       entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh'
       id: tpg-sync
-      secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["checkout"]
       args:
           - 'tpg-sync'
@@ -42,7 +41,7 @@ steps:
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       id: tpg-push
       waitFor: ["tpg-sync", "build-magician-binary"]
       env:
@@ -56,22 +55,21 @@ steps:
 
     - name: 'gcr.io/cloud-builders/git'
       waitFor: ["tpg-push"]
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       entrypoint: 'bash'
       args:
           - -c
           - |
             if [ "$BRANCH_NAME" == "main" ]; then
-              git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpg-sync
+              git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpg-sync
             else
-              git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpg-sync-$BRANCH_NAME
+              git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpg-sync-$BRANCH_NAME
             fi
 
     # TPGB
     - name: 'gcr.io/graphite-docker-images/bash-plus'
       entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh'
       id: tpgb-sync
-      secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["checkout"]
       args:
           - 'tpgb-sync'
@@ -80,7 +78,7 @@ steps:
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       id: tpgb-push
       waitFor: ["tpgb-sync", "build-magician-binary"]
       env:
@@ -94,22 +92,21 @@ steps:
 
     - name: 'gcr.io/cloud-builders/git'
       waitFor: ["tpgb-push"]
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       entrypoint: 'bash'
       args:
           - -c
           - |
             if [ "$BRANCH_NAME" == "main" ]; then
-              git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpgb-sync
+              git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpgb-sync
             else
-              git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpgb-sync-$BRANCH_NAME
+              git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tpgb-sync-$BRANCH_NAME
             fi
 
     # TGC
     - name: 'gcr.io/graphite-docker-images/bash-plus'
       entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh'
       id: tgc-sync
-      secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["checkout"]
       args:
           - 'tgc-sync'
@@ -118,7 +115,7 @@ steps:
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       id: tgc-push
       waitFor: ["tgc-sync", "tpgb-push"]
       env:
@@ -132,22 +129,21 @@ steps:
 
     - name: 'gcr.io/cloud-builders/git'
       waitFor: ["tgc-push"]
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       entrypoint: 'bash'
       args:
           - -c
           - |
             if [ "$BRANCH_NAME" == "main" ]; then
-              git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tgc-sync
+              git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tgc-sync
             else
-              git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tgc-sync-$BRANCH_NAME
+              git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tgc-sync-$BRANCH_NAME
             fi
 
     # TF-OICS
     - name: 'gcr.io/graphite-docker-images/bash-plus'
       entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh'
       id: tf-oics-sync
-      secretEnv: ["GITHUB_TOKEN"]
       waitFor: ["checkout"]
       args:
           - 'tf-oics-sync'
@@ -156,7 +152,7 @@ steps:
 
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       id: tf-oics-push
       waitFor: ["tf-oics-sync", "build-magician-binary"]
       env:
@@ -170,20 +166,20 @@ steps:
 
     - name: 'gcr.io/cloud-builders/git'
       waitFor: ["tf-oics-push"]
-      secretEnv: ["GITHUB_TOKEN"]
+      secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       entrypoint: 'bash'
       args:
           - -c
           - |
             if [ "$BRANCH_NAME" == "main" ]; then
-              git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tf-oics-sync
+              git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tf-oics-sync
             else
-              git push https://modular-magician:$$GITHUB_TOKEN@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tf-oics-sync-$BRANCH_NAME
+              git push https://modular-magician:$$GITHUB_TOKEN_CLASSIC@github.com/GoogleCloudPlatform/magic-modules $COMMIT_SHA:tf-oics-sync-$BRANCH_NAME
             fi
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
       entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh'
-      secretEnv: ["GITHUB_TOKEN", "GOOGLE_PROJECT"]
+      secretEnv: ["GITHUB_TOKEN_CLASSIC", "GOOGLE_PROJECT"]
       id: vcr-merge
       waitFor: ["tpg-push"]
       env:
@@ -196,7 +192,7 @@ steps:
       waitFor: ["vcr-merge"]
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv:
-        - "GITHUB_TOKEN"
+        - "GITHUB_TOKEN_DOWNSTREAMS"
         - "GOOGLE_BILLING_ACCOUNT"
         - "GOOGLE_CUST_ID"
         - "GOOGLE_FIRESTORE_PROJECT"
@@ -228,7 +224,9 @@ logsBucket: 'gs://cloudbuild-downstream-builder-logs'
 availableSecrets:
   secretManager:
     - versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest
-      env: GITHUB_TOKEN
+      env: GITHUB_TOKEN_CLASSIC
+    - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-downstreams/versions/latest
+      env: GITHUB_TOKEN_DOWNSTREAMS
     - versionName: projects/673497134629/secrets/ci-test-billing-account/versions/latest
       env: GOOGLE_BILLING_ACCOUNT
     - versionName: projects/673497134629/secrets/ci-test-cust-id/versions/latest

diff --git a/.ci/gcb-vcr-nightly.yml b/.ci/gcb-vcr-nightly.yml
@@ -42,4 +42,4 @@ availableSecrets:
     - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
       env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
     - versionName: projects/673497134629/secrets/ci-test-tpu-v2-vm-runtime-version/versions/latest
-      env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION
+      env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION
diff --git a/.ci/magician/cmd/check_cassettes.go b/.ci/magician/cmd/check_cassettes.go
@@ -13,7 +13,7 @@ import (
 
 var ccEnvironmentVariables = [...]string{
 	"COMMIT_SHA",
-	"GITHUB_TOKEN",
+	"GITHUB_TOKEN_DOWNSTREAMS",
 	"GOCACHE",
 	"GOPATH",
 	"GOOGLE_BILLING_ACCOUNT",
@@ -62,7 +62,7 @@ var checkCassettesCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN"], rnr)
+		ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN_DOWNSTREAMS"], rnr)
 
 		vt, err := vcr.NewTester(env, rnr)
 		if err != nil {

diff --git a/.ci/magician/cmd/community_checker.go b/.ci/magician/cmd/community_checker.go
@@ -64,7 +64,12 @@ var communityApprovalCmd = &cobra.Command{
 		baseBranch := args[5]
 		fmt.Println("Base Branch: ", baseBranch)
 
-		gh := github.NewClient()
+		githubToken, ok := os.LookupEnv("GITHUB_TOKEN_MAGIC_MODULES")
+		if !ok {
+			fmt.Println("Did not provide GITHUB_TOKEN_MAGIC_MODULES environment variable")
+			os.Exit(1)
+		}
+		gh := github.NewClient(githubToken)
 		cb := cloudbuild.NewClient()
 		execCommunityChecker(prNumber, commitSha, branchName, headRepoUrl, headBranch, baseBranch, gh, cb)
 	},