Skip to content

Commit

Permalink
temporarily disable failing e2e tests so we don't block all PRs.
Browse files Browse the repository at this point in the history 
Signed-off-by: Spencer Schrock <[email protected]>
  • Loading branch information
@spencerschrock
spencerschrock committed Jun 6, 2023
1 parent d961dda commit 3e1f3df
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 70 deletions.
142 changes: 72 additions & 70 deletions e2e/attestor_policy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,76 +111,78 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
},
expected: policy.Pass,
},
{
name: "test repo with simple code review requirements",
repoURL: "https://github.com/ossf/scorecard",
commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
MinReviewers: 1,
},
},
expected: policy.Pass,
},
{
name: "test code reviews required but repo doesn't have code reviews",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
PreventKnownVulnerabilities: true,
PreventUnpinnedDependencies: true,
EnsureCodeReviewed: true,
},
expected: policy.Fail,
},
{
name: "test code reviews required with min reviewers",
repoURL: "https://github.com/ossf/scorecard",
commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
PreventKnownVulnerabilities: false,
PreventUnpinnedDependencies: true,
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
MinReviewers: 1,
},
},
expected: policy.Pass,
},
{
name: "test code reviews required with min reviewers and required reviewers",
repoURL: "https://github.com/ossf/scorecard",
commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
PreventKnownVulnerabilities: false,
PreventUnpinnedDependencies: true,
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
MinReviewers: 1,
RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38"},
},
},
expected: policy.Pass,
},
{
name: "test code reviews required with too many min reviewers but matching required reviewers",
repoURL: "https://github.com/ossf/scorecard",
commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
PreventKnownVulnerabilities: false,
PreventUnpinnedDependencies: true,
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
MinReviewers: 2,
RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38"},
},
},
expected: policy.Fail,
},
// TODO(https://github.com/ossf/scorecard/issues/3129) temporarily skipping code review tests
//
// {
// name: "test repo with simple code review requirements",
// repoURL: "https://github.com/ossf/scorecard",
// commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
// policy: policy.AttestationPolicy{
// EnsureCodeReviewed: true,
// CodeReviewRequirements: policy.CodeReviewRequirements{
// MinReviewers: 1,
// },
// },
// expected: policy.Pass,
// },
// {
// name: "test code reviews required but repo doesn't have code reviews",
// repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
// policy: policy.AttestationPolicy{
// PreventBinaryArtifacts: true,
// PreventKnownVulnerabilities: true,
// PreventUnpinnedDependencies: true,
// EnsureCodeReviewed: true,
// },
// expected: policy.Fail,
// },
// {
// name: "test code reviews required with min reviewers",
// repoURL: "https://github.com/ossf/scorecard",
// commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
// policy: policy.AttestationPolicy{
// PreventBinaryArtifacts: true,
// PreventKnownVulnerabilities: false,
// PreventUnpinnedDependencies: true,
// EnsureCodeReviewed: true,
// CodeReviewRequirements: policy.CodeReviewRequirements{
// MinReviewers: 1,
// },
// },
// expected: policy.Pass,
// },
// {
// name: "test code reviews required with min reviewers and required reviewers",
// repoURL: "https://github.com/ossf/scorecard",
// commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
// policy: policy.AttestationPolicy{
// PreventBinaryArtifacts: true,
// PreventKnownVulnerabilities: false,
// PreventUnpinnedDependencies: true,
// EnsureCodeReviewed: true,
// CodeReviewRequirements: policy.CodeReviewRequirements{
// MinReviewers: 1,
// RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38"},
// },
// },
// expected: policy.Pass,
// },
// {
// name: "test code reviews required with too many min reviewers but matching required reviewers",
// repoURL: "https://github.com/ossf/scorecard",
// commit: "fa0592fab28aa92560f04e1ae8649dfff566ae2b",
// policy: policy.AttestationPolicy{
// PreventBinaryArtifacts: true,
// PreventKnownVulnerabilities: false,
// PreventUnpinnedDependencies: true,
// EnsureCodeReviewed: true,
// CodeReviewRequirements: policy.CodeReviewRequirements{
// MinReviewers: 2,
// RequiredApprovers: []string{"spencerschrock", "laurentsimon", "naveensrinivasan", "azeemshaikh38"},
// },
// },
// expected: policy.Fail,
// },
}

for _, tc := range tt {
Expand Down
1 change: 1 addition & 0 deletions e2e/ci_tests_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() {
Expect(repoClient.Close()).Should(BeNil())
})
It("Should return absence of CI tests in a repo with unsquashed merges", func() {
Skip("TODO(https://github.com/ossf/scorecard/issues/3129) temporarily skipping")
dl := scut.TestDetailLogger{}
repo, err := githubrepo.MakeGithubRepo("duo-labs/parliament")
Expect(err).Should(BeNil())
Expand Down

0 comments on commit 3e1f3df

Please sign in to comment.