Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🌱 speedup slowest e2e tests #3656

Merged
merged 3 commits into from
Nov 9, 2023
Merged

🌱 speedup slowest e2e tests #3656

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a3bae7a
switch ossfuzz test to smaller repo
spencerschrock Nov 8, 2023
0533810
reuse scorecard results for scorecard attestor policies
spencerschrock Nov 8, 2023
ba98ffc
Merge branch 'main' into e2e/fix-slow-tests
spencerschrock Nov 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 110 additions & 44 deletions e2e/attestor_policy_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
package e2e

import (
"context"
"fmt"
"os"
"strings"

Expand All @@ -24,11 +26,16 @@ import (

"github.com/ossf/scorecard/v4/attestor/command"
"github.com/ossf/scorecard/v4/attestor/policy"
"github.com/ossf/scorecard/v4/checker"
"github.com/ossf/scorecard/v4/checks"
"github.com/ossf/scorecard/v4/clients"
sclog "github.com/ossf/scorecard/v4/log"
"github.com/ossf/scorecard/v4/pkg"
)

var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
Context("E2E TEST:Validating scorecard attestation policy", func() {
It("Should attest to repos based on policy", func() {
It("Should attest to known good repos based on policy", func() {
tt := []struct {
name string
repoURL string
Expand All @@ -46,60 +53,85 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
},
expected: policy.Pass,
},
}

for _, tc := range tt {
f, err := os.CreateTemp("/tmp", strings.ReplaceAll(tc.name, " ", "-"))
Expect(err).Should(BeNil())
defer os.Remove(f.Name())

buf, err := yaml.Marshal(tc.policy)
Expect(err).Should(BeNil())

nbytes, err := f.Write(buf)
Expect(err).Should(BeNil())
Expect(nbytes).Should(BeNumerically(">", 0))

result, err := command.RunCheckWithParams(tc.repoURL, tc.commit, f.Name())
Expect(err).Should(BeNil())
Expect(result).Should(BeEquivalentTo(tc.expected))
}
})
})
})

var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
Context("E2E TEST:Validating scorecard attestation policy", func() {
It("Should attest to bad repos based on policy", func() {
tt := []struct {
name string
repoURL string
commit string
policy policy.AttestationPolicy
expected policy.PolicyResult
}{
{
name: "test bad repo with vulnerabilities prevented but no known vulnerabilities",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with vulnerabilities prevented but no known vulnerabilities",
policy: policy.AttestationPolicy{
PreventKnownVulnerabilities: true,
},
expected: policy.Pass,
},
{
name: "test bad repo with ignored binary artifact",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with ignored binary artifact",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
AllowedBinaryArtifacts: []string{"test-binary-artifact-*"},
},
expected: policy.Pass,
},
{
name: "test bad repo with binary artifact",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with binary artifact",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
},
expected: policy.Fail,
},
{
name: "test bad repo with ignored dep by path",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with ignored dep by path",
policy: policy.AttestationPolicy{
PreventUnpinnedDependencies: true,
AllowedUnpinnedDependencies: []policy.Dependency{{Filepath: "Dockerfile"}},
},
expected: policy.Pass,
},
{
name: "test bad repo without ignored dep",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo without ignored dep",
policy: policy.AttestationPolicy{
PreventUnpinnedDependencies: true,
},
expected: policy.Fail,
},
{
name: "test bad repo with ignored dep by name",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with ignored dep by name",
policy: policy.AttestationPolicy{
PreventUnpinnedDependencies: true,
AllowedUnpinnedDependencies: []policy.Dependency{{PackageName: "static-debian11"}, {PackageName: "golang"}},
},
expected: policy.Pass,
},
{
name: "test bad repo with everything ignored",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with everything ignored",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
AllowedBinaryArtifacts: []string{"test-binary-artifact-*"},
Expand All @@ -110,27 +142,46 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Pass,
},
{
name: "test repo with simple code review requirements",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required but repo doesn't have code reviews",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
MinReviewers: 1,
},
},
expected: policy.Pass,
expected: policy.Fail,
},
}
results, err := getScorecardResult("https://github.com/ossf-tests/scorecard-binauthz-test-bad")
Expect(err).Should(BeNil())
for _, tc := range tt {
got, err := tc.policy.EvaluateResults(&results.RawResults)
Expect(err).Should(BeNil())
Expect(got).Should(BeEquivalentTo(tc.expected))
}
})
})
})

var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
Context("E2E TEST:Validating scorecard attestation policy", func() {
It("Should attest to repos based on code review policy", func() {
tt := []struct {
name string
repoURL string
commit string
policy policy.AttestationPolicy
expected policy.PolicyResult
}{
{
name: "test code reviews required but repo doesn't have code reviews",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test repo with simple code review requirements",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
MinReviewers: 1,
},
},
expected: policy.Fail,
expected: policy.Pass,
},
{
name: "test code reviews required with min reviewers",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required with min reviewers",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
Expand All @@ -140,8 +191,7 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Pass,
},
{
name: "test code reviews required with min reviewers and required reviewers",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required with min reviewers and required reviewers",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
Expand All @@ -152,8 +202,7 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Pass,
},
{
name: "test code reviews required with too many min reviewers but matching required reviewers",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required with too many min reviewers but matching required reviewers",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
Expand All @@ -164,23 +213,40 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Fail,
},
}

results, err := getScorecardResult("https://github.com/ossf-tests/scorecard-attestor-code-review-e2e")
Expect(err).Should(BeNil())
for _, tc := range tt {
f, err := os.CreateTemp("/tmp", strings.ReplaceAll(tc.name, " ", "-"))
Expect(err).Should(BeNil())
defer os.Remove(f.Name())

buf, err := yaml.Marshal(tc.policy)
Expect(err).Should(BeNil())

nbytes, err := f.Write(buf)
got, err := tc.policy.EvaluateResults(&results.RawResults)
Expect(err).Should(BeNil())
Expect(nbytes).Should(BeNumerically(">", 0))

result, err := command.RunCheckWithParams(tc.repoURL, tc.commit, f.Name())
Expect(err).Should(BeNil())
Expect(result).Should(BeEquivalentTo(tc.expected))
Expect(got).Should(BeEquivalentTo(tc.expected))
}
})
})
})

func getScorecardResult(repoURL string) (pkg.ScorecardResult, error) {
ctx := context.Background()
logger := sclog.NewLogger(sclog.DefaultLevel)

enabledChecks := map[string]checker.Check{
checks.CheckBinaryArtifacts: {
Fn: checks.BinaryArtifacts,
},
checks.CheckVulnerabilities: {
Fn: checks.Vulnerabilities,
},
checks.CheckCodeReview: {
Fn: checks.CodeReview,
},
checks.CheckPinnedDependencies: {
Fn: checks.PinningDependencies,
},
}
repo, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, err := checker.GetClients(
ctx, repoURL, "", logger)
if err != nil {
return pkg.ScorecardResult{}, fmt.Errorf("couldn't set up clients: %w", err)
}
//nolint:wrapcheck,lll
return pkg.RunScorecard(ctx, repo, clients.HeadSHA, 0, enabledChecks, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient)
}
4 changes: 2 additions & 2 deletions e2e/fuzzing_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
Context("E2E TEST:Validating use of fuzzing tools", func() {
It("Should return use of OSS-Fuzz", func() {
dl := scut.TestDetailLogger{}
repo, err := githubrepo.MakeGithubRepo("tensorflow/tensorflow")
repo, err := githubrepo.MakeGithubRepo("ossf/scorecard-webapp")
Expect(err).Should(BeNil())
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
err = repoClient.InitRepo(repo, clients.HeadSHA, 0)
Expand All @@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 12,
NumberOfInfo: 3, // 1 for OSSFuzz, 2 for go native fuzzing
NumberOfDebug: 0,
}
result := checks.Fuzzing(&req)
Expand Down
Loading