diff --git a/e2e/attestor_policy_test.go b/e2e/attestor_policy_test.go index b11f2978374..9fe2c7b7ed2 100644 --- a/e2e/attestor_policy_test.go +++ b/e2e/attestor_policy_test.go @@ -15,6 +15,8 @@ package e2e import ( + "context" + "fmt" "os" "strings" @@ -24,11 +26,16 @@ import ( "github.com/ossf/scorecard/v4/attestor/command" "github.com/ossf/scorecard/v4/attestor/policy" + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/checks" + "github.com/ossf/scorecard/v4/clients" + sclog "github.com/ossf/scorecard/v4/log" + "github.com/ossf/scorecard/v4/pkg" ) var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { Context("E2E TEST:Validating scorecard attestation policy", func() { - It("Should attest to repos based on policy", func() { + It("Should attest to known good repos based on policy", func() { tt := []struct { name string repoURL string @@ -46,17 +53,47 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { }, expected: policy.Pass, }, + } + + for _, tc := range tt { + f, err := os.CreateTemp("/tmp", strings.ReplaceAll(tc.name, " ", "-")) + Expect(err).Should(BeNil()) + defer os.Remove(f.Name()) + + buf, err := yaml.Marshal(tc.policy) + Expect(err).Should(BeNil()) + + nbytes, err := f.Write(buf) + Expect(err).Should(BeNil()) + Expect(nbytes).Should(BeNumerically(">", 0)) + + result, err := command.RunCheckWithParams(tc.repoURL, tc.commit, f.Name()) + Expect(err).Should(BeNil()) + Expect(result).Should(BeEquivalentTo(tc.expected)) + } + }) + }) +}) + +var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { + Context("E2E TEST:Validating scorecard attestation policy", func() { + It("Should attest to bad repos based on policy", func() { + tt := []struct { + name string + repoURL string + commit string + policy policy.AttestationPolicy + expected policy.PolicyResult + }{ { - name: "test bad repo with vulnerabilities prevented but no known vulnerabilities", - repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", + name: "test bad repo with vulnerabilities prevented but no known vulnerabilities", policy: policy.AttestationPolicy{ PreventKnownVulnerabilities: true, }, expected: policy.Pass, }, { - name: "test bad repo with ignored binary artifact", - repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", + name: "test bad repo with ignored binary artifact", policy: policy.AttestationPolicy{ PreventBinaryArtifacts: true, AllowedBinaryArtifacts: []string{"test-binary-artifact-*"}, @@ -64,16 +101,14 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { expected: policy.Pass, }, { - name: "test bad repo with binary artifact", - repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", + name: "test bad repo with binary artifact", policy: policy.AttestationPolicy{ PreventBinaryArtifacts: true, }, expected: policy.Fail, }, { - name: "test bad repo with ignored dep by path", - repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", + name: "test bad repo with ignored dep by path", policy: policy.AttestationPolicy{ PreventUnpinnedDependencies: true, AllowedUnpinnedDependencies: []policy.Dependency{{Filepath: "Dockerfile"}}, @@ -81,16 +116,14 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { expected: policy.Pass, }, { - name: "test bad repo without ignored dep", - repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", + name: "test bad repo without ignored dep", policy: policy.AttestationPolicy{ PreventUnpinnedDependencies: true, }, expected: policy.Fail, }, { - name: "test bad repo with ignored dep by name", - repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", + name: "test bad repo with ignored dep by name", policy: policy.AttestationPolicy{ PreventUnpinnedDependencies: true, AllowedUnpinnedDependencies: []policy.Dependency{{PackageName: "static-debian11"}, {PackageName: "golang"}}, @@ -98,8 +131,7 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { expected: policy.Pass, }, { - name: "test bad repo with everything ignored", - repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", + name: "test bad repo with everything ignored", policy: policy.AttestationPolicy{ PreventBinaryArtifacts: true, AllowedBinaryArtifacts: []string{"test-binary-artifact-*"}, @@ -110,27 +142,46 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { expected: policy.Pass, }, { - name: "test repo with simple code review requirements", - repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e", + name: "test code reviews required but repo doesn't have code reviews", policy: policy.AttestationPolicy{ EnsureCodeReviewed: true, - CodeReviewRequirements: policy.CodeReviewRequirements{ - MinReviewers: 1, - }, }, - expected: policy.Pass, + expected: policy.Fail, }, + } + results, err := getScorecardResult("https://github.com/ossf-tests/scorecard-binauthz-test-bad") + Expect(err).Should(BeNil()) + for _, tc := range tt { + got, err := tc.policy.EvaluateResults(&results.RawResults) + Expect(err).Should(BeNil()) + Expect(got).Should(BeEquivalentTo(tc.expected)) + } + }) + }) +}) + +var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { + Context("E2E TEST:Validating scorecard attestation policy", func() { + It("Should attest to repos based on code review policy", func() { + tt := []struct { + name string + repoURL string + commit string + policy policy.AttestationPolicy + expected policy.PolicyResult + }{ { - name: "test code reviews required but repo doesn't have code reviews", - repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad", + name: "test repo with simple code review requirements", policy: policy.AttestationPolicy{ EnsureCodeReviewed: true, + CodeReviewRequirements: policy.CodeReviewRequirements{ + MinReviewers: 1, + }, }, - expected: policy.Fail, + expected: policy.Pass, }, { - name: "test code reviews required with min reviewers", - repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e", + name: "test code reviews required with min reviewers", policy: policy.AttestationPolicy{ EnsureCodeReviewed: true, CodeReviewRequirements: policy.CodeReviewRequirements{ @@ -140,8 +191,7 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { expected: policy.Pass, }, { - name: "test code reviews required with min reviewers and required reviewers", - repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e", + name: "test code reviews required with min reviewers and required reviewers", policy: policy.AttestationPolicy{ EnsureCodeReviewed: true, CodeReviewRequirements: policy.CodeReviewRequirements{ @@ -152,8 +202,7 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { expected: policy.Pass, }, { - name: "test code reviews required with too many min reviewers but matching required reviewers", - repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e", + name: "test code reviews required with too many min reviewers but matching required reviewers", policy: policy.AttestationPolicy{ EnsureCodeReviewed: true, CodeReviewRequirements: policy.CodeReviewRequirements{ @@ -164,23 +213,40 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() { expected: policy.Fail, }, } - + results, err := getScorecardResult("https://github.com/ossf-tests/scorecard-attestor-code-review-e2e") + Expect(err).Should(BeNil()) for _, tc := range tt { - f, err := os.CreateTemp("/tmp", strings.ReplaceAll(tc.name, " ", "-")) - Expect(err).Should(BeNil()) - defer os.Remove(f.Name()) - - buf, err := yaml.Marshal(tc.policy) - Expect(err).Should(BeNil()) - - nbytes, err := f.Write(buf) + got, err := tc.policy.EvaluateResults(&results.RawResults) Expect(err).Should(BeNil()) - Expect(nbytes).Should(BeNumerically(">", 0)) - - result, err := command.RunCheckWithParams(tc.repoURL, tc.commit, f.Name()) - Expect(err).Should(BeNil()) - Expect(result).Should(BeEquivalentTo(tc.expected)) + Expect(got).Should(BeEquivalentTo(tc.expected)) } }) }) }) + +func getScorecardResult(repoURL string) (pkg.ScorecardResult, error) { + ctx := context.Background() + logger := sclog.NewLogger(sclog.DefaultLevel) + + enabledChecks := map[string]checker.Check{ + checks.CheckBinaryArtifacts: { + Fn: checks.BinaryArtifacts, + }, + checks.CheckVulnerabilities: { + Fn: checks.Vulnerabilities, + }, + checks.CheckCodeReview: { + Fn: checks.CodeReview, + }, + checks.CheckPinnedDependencies: { + Fn: checks.PinningDependencies, + }, + } + repo, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, err := checker.GetClients( + ctx, repoURL, "", logger) + if err != nil { + return pkg.ScorecardResult{}, fmt.Errorf("couldn't set up clients: %w", err) + } + //nolint:wrapcheck,lll + return pkg.RunScorecard(ctx, repo, clients.HeadSHA, 0, enabledChecks, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient) +} diff --git a/e2e/fuzzing_test.go b/e2e/fuzzing_test.go index 83d6e720123..25935731092 100644 --- a/e2e/fuzzing_test.go +++ b/e2e/fuzzing_test.go @@ -34,7 +34,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { Context("E2E TEST:Validating use of fuzzing tools", func() { It("Should return use of OSS-Fuzz", func() { dl := scut.TestDetailLogger{} - repo, err := githubrepo.MakeGithubRepo("tensorflow/tensorflow") + repo, err := githubrepo.MakeGithubRepo("ossf/scorecard-webapp") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) err = repoClient.InitRepo(repo, clients.HeadSHA, 0) @@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { Error: nil, Score: checker.MaxResultScore, NumberOfWarn: 0, - NumberOfInfo: 12, + NumberOfInfo: 3, // 1 for OSSFuzz, 2 for go native fuzzing NumberOfDebug: 0, } result := checks.Fuzzing(&req)