From a5332ec8176373c73c7ee605197a312415ba27e5 Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Fri, 9 Jun 2023 11:44:34 -0700 Subject: [PATCH] :seedling: Re-enable skipped e2e tests. Switch to smaller code review repo. (#3144) * re-enable skipped ci test Signed-off-by: Spencer Schrock * re-enable skipped attestor test. switch to ossf-tests repo Signed-off-by: Spencer Schrock * remove extra policies from tests that only look at code review. Signed-off-by: Spencer Schrock * remove unneeded policies from binary artifact tests. Signed-off-by: Spencer Schrock --------- Signed-off-by: Spencer Schrock Signed-off-by: Avishay --- e2e/attestor_policy_test.go | 136 +++++++++++++++--------------------- e2e/ci_tests_test.go | 1 - 2 files changed, 58 insertions(+), 79 deletions(-) diff --git a/e2e/attestor_policy_test.go b/e2e/attestor_policy_test.go index e0837b76ada..b11f2978374 100644 --- a/e2e/attestor_policy_test.go +++ b/e2e/attestor_policy_test.go @@ -58,18 +58,16 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { name: "test bad repo with ignored binary artifact", repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", policy: policy.AttestationPolicy{ - PreventBinaryArtifacts: true, - AllowedBinaryArtifacts: []string{"test-binary-artifact-*"}, - PreventKnownVulnerabilities: true, + PreventBinaryArtifacts: true, + AllowedBinaryArtifacts: []string{"test-binary-artifact-*"}, }, expected: policy.Pass, }, { - name: "test bad repo with ignored binary artifact", + name: "test bad repo with binary artifact", repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", policy: policy.AttestationPolicy{ - PreventBinaryArtifacts: true, - PreventKnownVulnerabilities: true, + PreventBinaryArtifacts: true, }, expected: policy.Fail, }, @@ -111,78 +109,60 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { }, expected: policy.Pass, }, - // TODO(https://github.com/ossf/scorecard/issues/3129) temporarily skipping code review tests - // - // { - // name: "test repo with simple code review requirements", - // repoURL: "https://github.com/ossf/scorecard", - // commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b", - // policy: policy.AttestationPolicy{ - // EnsureCodeReviewed: true, - // CodeReviewRequirements: policy.CodeReviewRequirements{ - // MinReviewers: 1, - // }, - // }, - // expected: policy.Pass, - // }, - // { - // name: "test code reviews required but repo doesn't have code reviews", - // repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", - // policy: policy.AttestationPolicy{ - // PreventBinaryArtifacts: true, - // PreventKnownVulnerabilities: true, - // PreventUnpinnedDependencies: true, - // EnsureCodeReviewed: true, - // }, - // expected: policy.Fail, - // }, - // { - // name: "test code reviews required with min reviewers", - // repoURL: "https://github.com/ossf/scorecard", - // commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b", - // policy: policy.AttestationPolicy{ - // PreventBinaryArtifacts: true, - // PreventKnownVulnerabilities: false, - // PreventUnpinnedDependencies: true, - // EnsureCodeReviewed: true, - // CodeReviewRequirements: policy.CodeReviewRequirements{ - // MinReviewers: 1, - // }, - // }, - // expected: policy.Pass, - // }, - // { - // name: "test code reviews required with min reviewers and required reviewers", - // repoURL: "https://github.com/ossf/scorecard", - // commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b", - // policy: policy.AttestationPolicy{ - // PreventBinaryArtifacts: true, - // PreventKnownVulnerabilities: false, - // PreventUnpinnedDependencies: true, - // EnsureCodeReviewed: true, - // CodeReviewRequirements: policy.CodeReviewRequirements{ - // MinReviewers: 1, - // RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38"}, - // }, - // }, - // expected: policy.Pass, - // }, - // { - // name: "test code reviews required with too many min reviewers but matching required reviewers", - // repoURL: "https://github.com/ossf/scorecard", - // commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b", - // policy: policy.AttestationPolicy{ - // PreventBinaryArtifacts: true, - // PreventKnownVulnerabilities: false, - // PreventUnpinnedDependencies: true, - // EnsureCodeReviewed: true, - // CodeReviewRequirements: policy.CodeReviewRequirements{ - // MinReviewers: 2, - // RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38"}, - // }, - // }, - // expected: policy.Fail, - // }, + { + name: "test repo with simple code review requirements", + repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e", + policy: policy.AttestationPolicy{ + EnsureCodeReviewed: true, + CodeReviewRequirements: policy.CodeReviewRequirements{ + MinReviewers: 1, + }, + }, + expected: policy.Pass, + }, + { + name: "test code reviews required but repo doesn't have code reviews", + repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", + policy: policy.AttestationPolicy{ + EnsureCodeReviewed: true, + }, + expected: policy.Fail, + }, + { + name: "test code reviews required with min reviewers", + repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e", + policy: policy.AttestationPolicy{ + EnsureCodeReviewed: true, + CodeReviewRequirements: policy.CodeReviewRequirements{ + MinReviewers: 1, + }, + }, + expected: policy.Pass, + }, + { + name: "test code reviews required with min reviewers and required reviewers", + repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e", + policy: policy.AttestationPolicy{ + EnsureCodeReviewed: true, + CodeReviewRequirements: policy.CodeReviewRequirements{ + MinReviewers: 1, + RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38", "raghavkaul"}, + }, + }, + expected: policy.Pass, + }, + { + name: "test code reviews required with too many min reviewers but matching required reviewers", + repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e", + policy: policy.AttestationPolicy{ + EnsureCodeReviewed: true, + CodeReviewRequirements: policy.CodeReviewRequirements{ + MinReviewers: 2, + RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38", "raghavkaul"}, + }, + }, + expected: policy.Fail, + }, } for _, tc := range tt { diff --git a/e2e/ci_tests_test.go b/e2e/ci_tests_test.go index 30ff953c192..9730e4e432a 100644 --- a/e2e/ci_tests_test.go +++ b/e2e/ci_tests_test.go @@ -79,7 +79,6 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { Expect(repoClient.Close()).Should(BeNil()) }) It("Should return absence of CI tests in a repo with unsquashed merges", func() { - Skip("TODO(https://github.com/ossf/scorecard/issues/3129) temporarily skipping") dl := scut.TestDetailLogger{} repo, err := githubrepo.MakeGithubRepo("duo-labs/parliament") Expect(err).Should(BeNil())