e2e for GITHUB_TOKEN

ossf · Apr 12, 2022 · 2873c0d · 2873c0d
1 parent a46313c
commit 2873c0d
Show file tree

Hide file tree

Showing 4 changed files with 119 additions and 7 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -74,15 +74,89 @@ jobs:
         run: |
             go mod download
 
-      - name: Run E2E  #using retry because the GitHub token is being throttled. 
+      - name: Run PAT E2E  #using retry because the GitHub token is being throttled. 
         uses: nick-invision/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c 
         env:
           GITHUB_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }}
         with:
           max_attempts: 3
           retry_on: error
           timeout_minutes: 30
-          command: make e2e
+          command: make e2e-pat
+
+      - name: codecov
+        uses: codecov/codecov-action@e3c560433a6cc60aec8812599b7844a7b4fa0d71 # 2.1.0
+        with:
+         files: ./e2e-coverage.out
+         verbose: true
+
+      - name: find comment
+        uses: peter-evans/find-comment@1769778a0c5bd330272d749d12c036d65e70d39d # v1.2.0
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number || github.event.client_payload.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Integration tests ran for
+
+      - name: create or update comment
+        uses: peter-evans/create-or-update-comment@c9fcb64660bc90ec1cc535646af190c992007c32 # v1.4.5
+        with:
+          issue-number: ${{ github.event.pull_request.number || github.event.client_payload.pull_request.number }}
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body: |
+            Integration tests ${{ job.status }} for
+            [${{ github.event.client_payload.slash_command.args.named.sha || github.event.pull_request.head.sha }}]
+            (https://github.com/ossf/scorecard/actions/runs/${{ github.run_id }})
+
+  integration-gh-token:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@bdb12b622a910dfdc99a31fdfe6f45a16bc287a4 # v1
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+      - name: pull_request actions/checkout
+        uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # v2.3.4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: setup-go
+        uses: actions/setup-go@f6164bd8c8acb4a71fb2791a8b6c4024ff038dab # v2.2.0
+        with:
+          go-version: '1.17'
+
+      - name: Cache builds
+        # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds
+        uses: actions/cache@48af2dc4a9e8278b89d7fa154b955c30c6aaab09 #v2.1.7
+        with:
+          # In order:
+          # * Module download cache
+          # * Build cache (Linux)
+          # * Build cache (Mac)
+          # * Build cache (Windows)
+         path: |
+           ~/go/pkg/mod
+           ~/.cache/go-build
+           ~/Library/Caches/go-build
+           %LocalAppData%\go-build
+         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+         restore-keys: |
+           ${{ runner.os }}-go-
+
+      - name: Prepare test env
+        run: |
+            go mod download
+
+      - name: Run GITHUB_TOKEN E2E  #using retry because the GitHub token is being throttled. 
+        uses: nick-invision/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c 
+        env:
+          GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          max_attempts: 3
+          retry_on: error
+          timeout_minutes: 30
+          command: make e2e-gh-token
 
       - name: codecov
         uses: codecov/codecov-action@e3c560433a6cc60aec8812599b7844a7b4fa0d71 # 2.1.0

diff --git a/Makefile b/Makefile
@@ -277,7 +277,7 @@ cron-github-server-docker:
 
 ##@ Tests
 ################################# make test ###################################
-test-targets = unit-test e2e ci-e2e
+test-targets = unit-test e2e-pat e2e-gh-token ci-e2e
 .PHONY: test $(test-targets)
 test: $(test-targets)
 
@@ -293,8 +293,13 @@ ifndef GITHUB_AUTH_TOKEN
 	$(error GITHUB_AUTH_TOKEN is undefined)
 endif
 
-e2e: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token
-e2e: build-scorecard check-env | $(GINKGO)
+e2e-pat: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token
+e2e-pat: build-scorecard check-env | $(GINKGO)
 	# Run e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this
-	$(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out ./...
+	TOKEN_TYPE="PAT" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out ./...
+
+e2e-gh-token: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to default GITHUB_TOKEN
+e2e-gh-token: build-scorecard check-env | $(GINKGO)
+	# Run e2e tests. GITHUB_AUTH_TOKEN set to secrets.GITHUB_TOKEN must be used to run this.
+	TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out ./...
 ###############################################################################
diff --git a/e2e/branch_protection_test.go b/e2e/branch_protection_test.go
@@ -27,8 +27,10 @@ import (
 	scut "github.com/ossf/scorecard/v4/utests"
 )
 
-var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() {
+var _ = Describe("E2E TEST PAT:"+checks.CheckBranchProtection, func() {
 	Context("E2E TEST:Validating branch protection", func() {
+		skipIfTokenIs(githubWorkflowDefaultTokenType, checks.CheckBranchProtection+" does not support GITHUB_TOKEN")
+
 		It("Should get non-admin branch protection on other repositories", func() {
 			dl := scut.TestDetailLogger{}
 			repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e")

diff --git a/e2e/e2e_suite_test.go b/e2e/e2e_suite_test.go
@@ -15,6 +15,7 @@
 package e2e
 
 import (
+	"fmt"
 	"os"
 	"testing"
 
@@ -35,6 +36,36 @@ func TestE2e(t *testing.T) {
 
 var logger *log.Logger
 
+type tokenType int
+
+const (
+	patTokenType tokenType = iota
+	githubWorkflowDefaultTokenType
+)
+
+var tokType tokenType
+
+func skipIfTokenIs(t tokenType, msg string) {
+	if tokType == t {
+		Skip(msg)
+	}
+}
+
+func skipIfTokenIsNot(t tokenType, msg string) {
+	if tokType != t {
+		Skip(msg)
+	}
+}
+
 var _ = BeforeSuite(func() {
 	logger = log.NewLogger(log.DebugLevel)
+	tt := os.Getenv("TOKEN_TYPE")
+	switch tt {
+	case "PAT":
+		tokType = patTokenType
+	case "GITHUB_TOKEN":
+		tokType = githubWorkflowDefaultTokenType
+	default:
+		panic(fmt.Sprintf("invald TOKEN_TYPE: %s", tt))
+	}
 })