From 2873c0d58dfee453e436bf5c2dce2f9ee1c9d87c Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Tue, 12 Apr 2022 00:35:44 +0000 Subject: [PATCH] e2e for GITHUB_TOKEN --- .github/workflows/integration.yml | 78 ++++++++++++++++++++++++++++++- Makefile | 13 ++++-- e2e/branch_protection_test.go | 4 +- e2e/e2e_suite_test.go | 31 ++++++++++++ 4 files changed, 119 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1e487099351..c3e8710f74c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -74,7 +74,7 @@ jobs: run: | go mod download - - name: Run E2E #using retry because the GitHub token is being throttled. + - name: Run PAT E2E #using retry because the GitHub token is being throttled. uses: nick-invision/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c env: GITHUB_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }} @@ -82,7 +82,81 @@ jobs: max_attempts: 3 retry_on: error timeout_minutes: 30 - command: make e2e + command: make e2e-pat + + - name: codecov + uses: codecov/codecov-action@e3c560433a6cc60aec8812599b7844a7b4fa0d71 # 2.1.0 + with: + files: ./e2e-coverage.out + verbose: true + + - name: find comment + uses: peter-evans/find-comment@1769778a0c5bd330272d749d12c036d65e70d39d # v1.2.0 + id: fc + with: + issue-number: ${{ github.event.pull_request.number || github.event.client_payload.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: Integration tests ran for + + - name: create or update comment + uses: peter-evans/create-or-update-comment@c9fcb64660bc90ec1cc535646af190c992007c32 # v1.4.5 + with: + issue-number: ${{ github.event.pull_request.number || github.event.client_payload.pull_request.number }} + comment-id: ${{ steps.fc.outputs.comment-id }} + body: | + Integration tests ${{ job.status }} for + [${{ github.event.client_payload.slash_command.args.named.sha || github.event.pull_request.head.sha }}] + (https://github.com/ossf/scorecard/actions/runs/${{ github.run_id }}) + + integration-gh-token: + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@bdb12b622a910dfdc99a31fdfe6f45a16bc287a4 # v1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + + - name: pull_request actions/checkout + uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 # v2.3.4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: setup-go + uses: actions/setup-go@f6164bd8c8acb4a71fb2791a8b6c4024ff038dab # v2.2.0 + with: + go-version: '1.17' + + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@48af2dc4a9e8278b89d7fa154b955c30c6aaab09 #v2.1.7 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + %LocalAppData%\go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Prepare test env + run: | + go mod download + + - name: Run GITHUB_TOKEN E2E #using retry because the GitHub token is being throttled. + uses: nick-invision/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c + env: + GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + max_attempts: 3 + retry_on: error + timeout_minutes: 30 + command: make e2e-gh-token - name: codecov uses: codecov/codecov-action@e3c560433a6cc60aec8812599b7844a7b4fa0d71 # 2.1.0 diff --git a/Makefile b/Makefile index 9fdfd19641a..cba83c4877e 100644 --- a/Makefile +++ b/Makefile @@ -277,7 +277,7 @@ cron-github-server-docker: ##@ Tests ################################# make test ################################### -test-targets = unit-test e2e ci-e2e +test-targets = unit-test e2e-pat e2e-gh-token ci-e2e .PHONY: test $(test-targets) test: $(test-targets) @@ -293,8 +293,13 @@ ifndef GITHUB_AUTH_TOKEN $(error GITHUB_AUTH_TOKEN is undefined) endif -e2e: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token -e2e: build-scorecard check-env | $(GINKGO) +e2e-pat: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token +e2e-pat: build-scorecard check-env | $(GINKGO) # Run e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this - $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out ./... + TOKEN_TYPE="PAT" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out ./... + +e2e-gh-token: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to default GITHUB_TOKEN +e2e-gh-token: build-scorecard check-env | $(GINKGO) + # Run e2e tests. GITHUB_AUTH_TOKEN set to secrets.GITHUB_TOKEN must be used to run this. + TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -cover -coverprofile=e2e-coverage.out ./... ############################################################################### diff --git a/e2e/branch_protection_test.go b/e2e/branch_protection_test.go index 06e733db60f..fb56d749e6e 100644 --- a/e2e/branch_protection_test.go +++ b/e2e/branch_protection_test.go @@ -27,8 +27,10 @@ import ( scut "github.com/ossf/scorecard/v4/utests" ) -var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() { +var _ = Describe("E2E TEST PAT:"+checks.CheckBranchProtection, func() { Context("E2E TEST:Validating branch protection", func() { + skipIfTokenIs(githubWorkflowDefaultTokenType, checks.CheckBranchProtection+" does not support GITHUB_TOKEN") + It("Should get non-admin branch protection on other repositories", func() { dl := scut.TestDetailLogger{} repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e") diff --git a/e2e/e2e_suite_test.go b/e2e/e2e_suite_test.go index 0e587652ff4..5f81dfbc35e 100644 --- a/e2e/e2e_suite_test.go +++ b/e2e/e2e_suite_test.go @@ -15,6 +15,7 @@ package e2e import ( + "fmt" "os" "testing" @@ -35,6 +36,36 @@ func TestE2e(t *testing.T) { var logger *log.Logger +type tokenType int + +const ( + patTokenType tokenType = iota + githubWorkflowDefaultTokenType +) + +var tokType tokenType + +func skipIfTokenIs(t tokenType, msg string) { + if tokType == t { + Skip(msg) + } +} + +func skipIfTokenIsNot(t tokenType, msg string) { + if tokType != t { + Skip(msg) + } +} + var _ = BeforeSuite(func() { logger = log.NewLogger(log.DebugLevel) + tt := os.Getenv("TOKEN_TYPE") + switch tt { + case "PAT": + tokType = patTokenType + case "GITHUB_TOKEN": + tokType = githubWorkflowDefaultTokenType + default: + panic(fmt.Sprintf("invald TOKEN_TYPE: %s", tt)) + } })