Skip to content

Commit

Permalink
reuse scorecard results for scorecard attestor policies
Browse files Browse the repository at this point in the history 
previously this test took 27 seconds locally, and now takes 8.
which is split across 3 subtests:
good repos: 1s
bad repos: 5s
code review policies: 2s

Signed-off-by: Spencer Schrock <[email protected]>
  • Loading branch information
@spencerschrock
spencerschrock committed Nov 8, 2023
1 parent a3bae7a commit 0533810
Showing 1 changed file with 110 additions and 44 deletions.
154 changes: 110 additions & 44 deletions e2e/attestor_policy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
package e2e

import (
"context"
"fmt"
"os"
"strings"

Expand All @@ -24,11 +26,16 @@ import (

"github.com/ossf/scorecard/v4/attestor/command"
"github.com/ossf/scorecard/v4/attestor/policy"
"github.com/ossf/scorecard/v4/checker"
"github.com/ossf/scorecard/v4/checks"
"github.com/ossf/scorecard/v4/clients"
sclog "github.com/ossf/scorecard/v4/log"
"github.com/ossf/scorecard/v4/pkg"
)

var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
Context("E2E TEST:Validating scorecard attestation policy", func() {
It("Should attest to repos based on policy", func() {
It("Should attest to known good repos based on policy", func() {
tt := []struct {
name string
repoURL string
Expand All @@ -46,60 +53,85 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
},
expected: policy.Pass,
},
}

for _, tc := range tt {
f, err := os.CreateTemp("/tmp", strings.ReplaceAll(tc.name, " ", "-"))
Expect(err).Should(BeNil())
defer os.Remove(f.Name())

buf, err := yaml.Marshal(tc.policy)
Expect(err).Should(BeNil())

nbytes, err := f.Write(buf)
Expect(err).Should(BeNil())
Expect(nbytes).Should(BeNumerically(">", 0))

result, err := command.RunCheckWithParams(tc.repoURL, tc.commit, f.Name())
Expect(err).Should(BeNil())
Expect(result).Should(BeEquivalentTo(tc.expected))
}
})
})
})

var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
Context("E2E TEST:Validating scorecard attestation policy", func() {
It("Should attest to bad repos based on policy", func() {
tt := []struct {
name string
repoURL string
commit string
policy policy.AttestationPolicy
expected policy.PolicyResult
}{
{
name: "test bad repo with vulnerabilities prevented but no known vulnerabilities",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with vulnerabilities prevented but no known vulnerabilities",
policy: policy.AttestationPolicy{
PreventKnownVulnerabilities: true,
},
expected: policy.Pass,
},
{
name: "test bad repo with ignored binary artifact",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with ignored binary artifact",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
AllowedBinaryArtifacts: []string{"test-binary-artifact-*"},
},
expected: policy.Pass,
},
{
name: "test bad repo with binary artifact",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with binary artifact",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
},
expected: policy.Fail,
},
{
name: "test bad repo with ignored dep by path",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with ignored dep by path",
policy: policy.AttestationPolicy{
PreventUnpinnedDependencies: true,
AllowedUnpinnedDependencies: []policy.Dependency{{Filepath: "Dockerfile"}},
},
expected: policy.Pass,
},
{
name: "test bad repo without ignored dep",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo without ignored dep",
policy: policy.AttestationPolicy{
PreventUnpinnedDependencies: true,
},
expected: policy.Fail,
},
{
name: "test bad repo with ignored dep by name",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with ignored dep by name",
policy: policy.AttestationPolicy{
PreventUnpinnedDependencies: true,
AllowedUnpinnedDependencies: []policy.Dependency{{PackageName: "static-debian11"}, {PackageName: "golang"}},
},
expected: policy.Pass,
},
{
name: "test bad repo with everything ignored",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with everything ignored",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
AllowedBinaryArtifacts: []string{"test-binary-artifact-*"},
Expand All @@ -110,27 +142,46 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Pass,
},
{
name: "test repo with simple code review requirements",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required but repo doesn't have code reviews",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
MinReviewers: 1,
},
},
expected: policy.Pass,
expected: policy.Fail,
},
}
results, err := getScorecardResult("https://github.com/ossf-tests/scorecard-binauthz-test-bad")
Expect(err).Should(BeNil())
for _, tc := range tt {
got, err := tc.policy.EvaluateResults(&results.RawResults)
Expect(err).Should(BeNil())
Expect(got).Should(BeEquivalentTo(tc.expected))
}
})
})
})

var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
Context("E2E TEST:Validating scorecard attestation policy", func() {
It("Should attest to repos based on code review policy", func() {
tt := []struct {
name string
repoURL string
commit string
policy policy.AttestationPolicy
expected policy.PolicyResult
}{
{
name: "test code reviews required but repo doesn't have code reviews",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test repo with simple code review requirements",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
MinReviewers: 1,
},
},
expected: policy.Fail,
expected: policy.Pass,
},
{
name: "test code reviews required with min reviewers",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required with min reviewers",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
Expand All @@ -140,8 +191,7 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Pass,
},
{
name: "test code reviews required with min reviewers and required reviewers",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required with min reviewers and required reviewers",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
Expand All @@ -152,8 +202,7 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Pass,
},
{
name: "test code reviews required with too many min reviewers but matching required reviewers",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required with too many min reviewers but matching required reviewers",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
Expand All @@ -164,23 +213,40 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Fail,
},
}

results, err := getScorecardResult("https://github.com/ossf-tests/scorecard-attestor-code-review-e2e")
Expect(err).Should(BeNil())
for _, tc := range tt {
f, err := os.CreateTemp("/tmp", strings.ReplaceAll(tc.name, " ", "-"))
Expect(err).Should(BeNil())
defer os.Remove(f.Name())

buf, err := yaml.Marshal(tc.policy)
Expect(err).Should(BeNil())

nbytes, err := f.Write(buf)
got, err := tc.policy.EvaluateResults(&results.RawResults)
Expect(err).Should(BeNil())
Expect(nbytes).Should(BeNumerically(">", 0))

result, err := command.RunCheckWithParams(tc.repoURL, tc.commit, f.Name())
Expect(err).Should(BeNil())
Expect(result).Should(BeEquivalentTo(tc.expected))
Expect(got).Should(BeEquivalentTo(tc.expected))
}
})
})
})

func getScorecardResult(repoURL string) (pkg.ScorecardResult, error) {
ctx := context.Background()
logger := sclog.NewLogger(sclog.DefaultLevel)

enabledChecks := map[string]checker.Check{
checks.CheckBinaryArtifacts: {
Fn: checks.BinaryArtifacts,
},
checks.CheckVulnerabilities: {
Fn: checks.Vulnerabilities,
},
checks.CheckCodeReview: {
Fn: checks.CodeReview,
},
checks.CheckPinnedDependencies: {
Fn: checks.PinningDependencies,
},
}
repo, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, err := checker.GetClients(
ctx, repoURL, "", logger)
if err != nil {
return pkg.ScorecardResult{}, fmt.Errorf("couldn't set up clients: %w", err)
}
//nolint:wrapcheck,lll
return pkg.RunScorecard(ctx, repo, clients.HeadSHA, 0, enabledChecks, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient)
}

0 comments on commit 0533810

Please sign in to comment.