🌱 Re-enable skipped e2e tests. Switch to smaller code review repo. (#…

…3144) * re-enable skipped ci test Signed-off-by: Spencer Schrock <[email protected]> * re-enable skipped attestor test. switch to ossf-tests repo Signed-off-by: Spencer Schrock <[email protected]> * remove extra policies from tests that only look at code review. Signed-off-by: Spencer Schrock <[email protected]> * remove unneeded policies from binary artifact tests. Signed-off-by: Spencer Schrock <[email protected]> --------- Signed-off-by: Spencer Schrock <[email protected]> Signed-off-by: Avishay <[email protected]>
ossf · Jun 11, 2023 · a5332ec · a5332ec
1 parent 48171a8
commit a5332ec
Show file tree

Hide file tree

Showing 2 changed files with 58 additions and 79 deletions.
diff --git a/e2e/attestor_policy_test.go b/e2e/attestor_policy_test.go
@@ -58,18 +58,16 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
 					name:    "test bad repo with ignored binary artifact",
 					repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
 					policy: policy.AttestationPolicy{
-						PreventBinaryArtifacts:      true,
-						AllowedBinaryArtifacts:      []string{"test-binary-artifact-*"},
-						PreventKnownVulnerabilities: true,
+						PreventBinaryArtifacts: true,
+						AllowedBinaryArtifacts: []string{"test-binary-artifact-*"},
 					},
 					expected: policy.Pass,
 				},
 				{
-					name:    "test bad repo with ignored binary artifact",
+					name:    "test bad repo with binary artifact",
 					repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
 					policy: policy.AttestationPolicy{
-						PreventBinaryArtifacts:      true,
-						PreventKnownVulnerabilities: true,
+						PreventBinaryArtifacts: true,
 					},
 					expected: policy.Fail,
 				},
@@ -111,78 +109,60 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
 					},
 					expected: policy.Pass,
 				},
-				// TODO(https://github.com/ossf/scorecard/issues/3129) temporarily skipping code review tests
-				//
-				// {
-				// 	name:    "test repo with simple code review requirements",
-				// 	repoURL: "https://github.com/ossf/scorecard",
-				// 	commit:  "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
-				// 	policy: policy.AttestationPolicy{
-				// 		EnsureCodeReviewed: true,
-				// 		CodeReviewRequirements: policy.CodeReviewRequirements{
-				// 			MinReviewers: 1,
-				// 		},
-				// 	},
-				// 	expected: policy.Pass,
-				// },
-				// {
-				// 	name:    "test code reviews required but repo doesn't have code reviews",
-				// 	repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
-				// 	policy: policy.AttestationPolicy{
-				// 		PreventBinaryArtifacts:      true,
-				// 		PreventKnownVulnerabilities: true,
-				// 		PreventUnpinnedDependencies: true,
-				// 		EnsureCodeReviewed:          true,
-				// 	},
-				// 	expected: policy.Fail,
-				// },
-				// {
-				// 	name:    "test code reviews required with min reviewers",
-				// 	repoURL: "https://github.com/ossf/scorecard",
-				// 	commit:  "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
-				// 	policy: policy.AttestationPolicy{
-				// 		PreventBinaryArtifacts:      true,
-				// 		PreventKnownVulnerabilities: false,
-				// 		PreventUnpinnedDependencies: true,
-				// 		EnsureCodeReviewed:          true,
-				// 		CodeReviewRequirements: policy.CodeReviewRequirements{
-				// 			MinReviewers: 1,
-				// 		},
-				// 	},
-				// 	expected: policy.Pass,
-				// },
-				// {
-				// 	name:    "test code reviews required with min reviewers and required reviewers",
-				// 	repoURL: "https://github.com/ossf/scorecard",
-				// 	commit:  "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
-				// 	policy: policy.AttestationPolicy{
-				// 		PreventBinaryArtifacts:      true,
-				// 		PreventKnownVulnerabilities: false,
-				// 		PreventUnpinnedDependencies: true,
-				// 		EnsureCodeReviewed:          true,
-				// 		CodeReviewRequirements: policy.CodeReviewRequirements{
-				// 			MinReviewers:      1,
-				// 			RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38"},
-				// 		},
-				// 	},
-				// 	expected: policy.Pass,
-				// },
-				// {
-				// 	name:    "test code reviews required with too many min reviewers but matching required reviewers",
-				// 	repoURL: "https://github.com/ossf/scorecard",
-				// 	commit:  "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
-				// 	policy: policy.AttestationPolicy{
-				// 		PreventBinaryArtifacts:      true,
-				// 		PreventKnownVulnerabilities: false,
-				// 		PreventUnpinnedDependencies: true,
-				// 		EnsureCodeReviewed:          true,
-				// 		CodeReviewRequirements: policy.CodeReviewRequirements{
-				// 			MinReviewers:      2,
-				// 			RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38"},
-				// 		},
-				// 	},
-				// 	expected: policy.Fail,
-				// },
+				{
+					name:    "test repo with simple code review requirements",
+					repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
+					policy: policy.AttestationPolicy{
+						EnsureCodeReviewed: true,
+						CodeReviewRequirements: policy.CodeReviewRequirements{
+							MinReviewers: 1,
+						},
+					},
+					expected: policy.Pass,
+				},
+				{
+					name:    "test code reviews required but repo doesn't have code reviews",
+					repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
+					policy: policy.AttestationPolicy{
+						EnsureCodeReviewed: true,
+					},
+					expected: policy.Fail,
+				},
+				{
+					name:    "test code reviews required with min reviewers",
+					repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
+					policy: policy.AttestationPolicy{
+						EnsureCodeReviewed: true,
+						CodeReviewRequirements: policy.CodeReviewRequirements{
+							MinReviewers: 1,
+						},
+					},
+					expected: policy.Pass,
+				},
+				{
+					name:    "test code reviews required with min reviewers and required reviewers",
+					repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
+					policy: policy.AttestationPolicy{
+						EnsureCodeReviewed: true,
+						CodeReviewRequirements: policy.CodeReviewRequirements{
+							MinReviewers:      1,
+							RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38", "raghavkaul"},
+						},
+					},
+					expected: policy.Pass,
+				},
+				{
+					name:    "test code reviews required with too many min reviewers but matching required reviewers",
+					repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
+					policy: policy.AttestationPolicy{
+						EnsureCodeReviewed: true,
+						CodeReviewRequirements: policy.CodeReviewRequirements{
+							MinReviewers:      2,
+							RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38", "raghavkaul"},
+						},
+					},
+					expected: policy.Fail,
+				},
 			}
 
 			for _, tc := range tt {

diff --git a/e2e/ci_tests_test.go b/e2e/ci_tests_test.go
@@ -79,7 +79,6 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
 			Expect(repoClient.Close()).Should(BeNil())
 		})
 		It("Should return absence of CI tests in a repo with unsquashed merges", func() {
-			Skip("TODO(https://github.com/ossf/scorecard/issues/3129) temporarily skipping")
 			dl := scut.TestDetailLogger{}
 			repo, err := githubrepo.MakeGithubRepo("duo-labs/parliament")
 			Expect(err).Should(BeNil())