From 1f12364da7a044851a0de779b44aa211c70e0295 Mon Sep 17 00:00:00 2001 From: Azeem Shaikh Date: Mon, 7 Feb 2022 17:15:58 -0800 Subject: [PATCH] Add support for commit-based Scorecard --- checker/check_request.go | 2 + checks/binary_artifact.go | 1 + checks/binary_artifact_test.go | 2 +- checks/ci_tests.go | 5 ++- checks/code_review.go | 5 ++- checks/dangerous_workflow.go | 1 + checks/license_test.go | 2 +- checks/permissions.go | 1 + checks/pinned_dependencies.go | 1 + checks/raw/security_policy.go | 2 +- checks/sast.go | 1 + checks/vulnerabilities.go | 5 ++- clients/githubrepo/client.go | 5 +-- clients/localdir/client.go | 2 +- clients/localdir/client_test.go | 2 +- clients/mockclients/repo_client.go | 8 ++-- clients/repo_client.go | 2 +- cmd/flags.go | 1 + cmd/root.go | 14 ++++++- cmd/serve.go | 2 +- cron/worker/main.go | 2 +- e2e/binary_artifacts_test.go | 64 ++++++++++++++++++++++++++++-- e2e/branch_protection_test.go | 6 +-- e2e/ci_tests_test.go | 26 +++++++++++- e2e/code_review_test.go | 27 ++++++++++++- e2e/contributors_test.go | 2 +- e2e/dangerous_workflow_test.go | 31 ++++++++++++++- e2e/dependency_update_tool_test.go | 4 +- e2e/fuzzing_test.go | 6 +-- e2e/license_test.go | 2 +- e2e/maintained_test.go | 2 +- e2e/packaging_test.go | 2 +- e2e/permissions_test.go | 32 ++++++++++++++- e2e/pinned_dependencies_test.go | 27 ++++++++++++- e2e/sast_test.go | 2 +- e2e/security_policy_test.go | 4 +- e2e/signedreleases_test.go | 2 +- e2e/vulnerabilities_test.go | 35 +++++++++++++++- pkg/scorecard.go | 3 +- 39 files changed, 297 insertions(+), 46 deletions(-) diff --git a/checker/check_request.go b/checker/check_request.go index 7266bb25aed..caaf66a08bf 100644 --- a/checker/check_request.go +++ b/checker/check_request.go @@ -40,6 +40,8 @@ type RequestType int const ( // FileBased request types require checks to run solely on file-content. FileBased RequestType = iota + // CommitBased request types require checks to run on non-HEAD commit content. + CommitBased ) // ListUnsupported returns []RequestType not in `supported` and are `required`. diff --git a/checks/binary_artifact.go b/checks/binary_artifact.go index 8439aee7a15..a3d0d567871 100644 --- a/checks/binary_artifact.go +++ b/checks/binary_artifact.go @@ -28,6 +28,7 @@ const CheckBinaryArtifacts string = "Binary-Artifacts" func init() { var supportedRequestTypes = []checker.RequestType{ checker.FileBased, + checker.CommitBased, } if err := registerCheck(CheckBinaryArtifacts, BinaryArtifacts, supportedRequestTypes); err != nil { // this should never happen diff --git a/checks/binary_artifact_test.go b/checks/binary_artifact_test.go index c82165e4145..3fe8a86efe3 100644 --- a/checks/binary_artifact_test.go +++ b/checks/binary_artifact_test.go @@ -72,7 +72,7 @@ func TestBinaryArtifacts(t *testing.T) { ctx := context.Background() client := localdir.CreateLocalDirClient(ctx, logger) - if err := client.InitRepo(repo); err != nil { + if err := client.InitRepo(repo, "HEAD"); err != nil { t.Errorf("InitRepo: %v", err) } diff --git a/checks/ci_tests.go b/checks/ci_tests.go index 8da124c143b..0c53dd6e2d9 100644 --- a/checks/ci_tests.go +++ b/checks/ci_tests.go @@ -31,7 +31,10 @@ const ( //nolint:gochecknoinits func init() { - if err := registerCheck(CheckCITests, CITests, nil); err != nil { + supportedRequestTypes := []checker.RequestType{ + checker.CommitBased, + } + if err := registerCheck(CheckCITests, CITests, supportedRequestTypes); err != nil { // this should never happen panic(err) } diff --git a/checks/code_review.go b/checks/code_review.go index 73365d7d551..91c6ad9aba2 100644 --- a/checks/code_review.go +++ b/checks/code_review.go @@ -26,7 +26,10 @@ const CheckCodeReview = "Code-Review" //nolint:gochecknoinits func init() { - if err := registerCheck(CheckCodeReview, CodeReview, nil); err != nil { + supportedRequestTypes := []checker.RequestType{ + checker.CommitBased, + } + if err := registerCheck(CheckCodeReview, CodeReview, supportedRequestTypes); err != nil { // this should never happen panic(err) } diff --git a/checks/dangerous_workflow.go b/checks/dangerous_workflow.go index e418cb80c88..e36f0168193 100644 --- a/checks/dangerous_workflow.go +++ b/checks/dangerous_workflow.go @@ -61,6 +61,7 @@ func containsUntrustedContextPattern(variable string) bool { func init() { supportedRequestTypes := []checker.RequestType{ checker.FileBased, + checker.CommitBased, } if err := registerCheck(CheckDangerousWorkflow, DangerousWorkflow, supportedRequestTypes); err != nil { // this should never happen diff --git a/checks/license_test.go b/checks/license_test.go index 8ac0985a21c..260e3f4a6d1 100644 --- a/checks/license_test.go +++ b/checks/license_test.go @@ -154,7 +154,7 @@ func TestLicenseFileSubdirectory(t *testing.T) { ctx := context.Background() client := localdir.CreateLocalDirClient(ctx, logger) - if err := client.InitRepo(repo); err != nil { + if err := client.InitRepo(repo, "HEAD"); err != nil { t.Errorf("InitRepo: %v", err) } diff --git a/checks/permissions.go b/checks/permissions.go index 7b4fdf8832e..9a7d88b3414 100644 --- a/checks/permissions.go +++ b/checks/permissions.go @@ -55,6 +55,7 @@ var permissionsOfInterest = []permission{ func init() { supportedRequestTypes := []checker.RequestType{ checker.FileBased, + checker.CommitBased, } if err := registerCheck(CheckTokenPermissions, TokenPermissions, supportedRequestTypes); err != nil { // This should never happen. diff --git a/checks/pinned_dependencies.go b/checks/pinned_dependencies.go index b74a00bef80..5b991a6a323 100644 --- a/checks/pinned_dependencies.go +++ b/checks/pinned_dependencies.go @@ -41,6 +41,7 @@ type worklowPinningResult struct { func init() { supportedRequestTypes := []checker.RequestType{ checker.FileBased, + checker.CommitBased, } if err := registerCheck(CheckPinnedDependencies, PinnedDependencies, supportedRequestTypes); err != nil { // This should never happen. diff --git a/checks/raw/security_policy.go b/checks/raw/security_policy.go index 2a171d522b3..6d43bfebcb8 100644 --- a/checks/raw/security_policy.go +++ b/checks/raw/security_policy.go @@ -80,7 +80,7 @@ func SecurityPolicy(c *checker.CheckRequest) (checker.SecurityPolicyData, error) Repo: c.Repo.Org(), } - err = dotGitHub.RepoClient.InitRepo(dotGitHub.Repo) + err = dotGitHub.RepoClient.InitRepo(dotGitHub.Repo, "HEAD") switch { case err == nil: defer dotGitHub.RepoClient.Close() diff --git a/checks/sast.go b/checks/sast.go index 7953b0ac297..e50969d4d5f 100644 --- a/checks/sast.go +++ b/checks/sast.go @@ -31,6 +31,7 @@ var allowedConclusions = map[string]bool{"success": true, "neutral": true} //nolint:gochecknoinits func init() { + // TODO(#575): Check if we can support commit-based requests here. if err := registerCheck(CheckSAST, SAST, nil); err != nil { // This should never happen. panic(err) diff --git a/checks/vulnerabilities.go b/checks/vulnerabilities.go index afffa6ecf6b..94706c49680 100644 --- a/checks/vulnerabilities.go +++ b/checks/vulnerabilities.go @@ -26,7 +26,10 @@ const CheckVulnerabilities = "Vulnerabilities" //nolint:gochecknoinits func init() { - if err := registerCheck(CheckVulnerabilities, Vulnerabilities, nil); err != nil { + supportedRequestTypes := []checker.RequestType{ + checker.CommitBased, + } + if err := registerCheck(CheckVulnerabilities, Vulnerabilities, supportedRequestTypes); err != nil { // this should never happen panic(err) } diff --git a/clients/githubrepo/client.go b/clients/githubrepo/client.go index 1a1557b6424..18bd83610dd 100644 --- a/clients/githubrepo/client.go +++ b/clients/githubrepo/client.go @@ -51,8 +51,7 @@ type Client struct { } // InitRepo sets up the GitHub repo in local storage for improving performance and GitHub token usage efficiency. -func (client *Client) InitRepo(inputRepo clients.Repo) error { - commitSHA := "HEAD" +func (client *Client) InitRepo(inputRepo clients.Repo, commitSHA string) error { ghRepo, ok := inputRepo.(*repoURL) if !ok { return fmt.Errorf("%w: %v", errInputRepoType, inputRepo) @@ -225,7 +224,7 @@ func CreateOssFuzzRepoClient(ctx context.Context, logger *log.Logger) (clients.R } ossFuzzRepoClient := CreateGithubRepoClient(ctx, logger) - if err := ossFuzzRepoClient.InitRepo(ossFuzzRepo); err != nil { + if err := ossFuzzRepoClient.InitRepo(ossFuzzRepo, "HEAD"); err != nil { return nil, fmt.Errorf("error during InitRepo: %w", err) } return ossFuzzRepoClient, nil diff --git a/clients/localdir/client.go b/clients/localdir/client.go index 6f0e76d44f0..029e4531f50 100644 --- a/clients/localdir/client.go +++ b/clients/localdir/client.go @@ -44,7 +44,7 @@ type localDirClient struct { } // InitRepo sets up the local repo. -func (client *localDirClient) InitRepo(inputRepo clients.Repo) error { +func (client *localDirClient) InitRepo(inputRepo clients.Repo, commitSHA string) error { localRepo, ok := inputRepo.(*repoLocal) if !ok { return fmt.Errorf("%w: %v", errInputRepoType, inputRepo) diff --git a/clients/localdir/client_test.go b/clients/localdir/client_test.go index c0baa592caf..0a3b786f456 100644 --- a/clients/localdir/client_test.go +++ b/clients/localdir/client_test.go @@ -75,7 +75,7 @@ func TestClient_CreationAndCaching(t *testing.T) { } client := CreateLocalDirClient(ctx, logger) - if err := client.InitRepo(repo); err != nil { + if err := client.InitRepo(repo, "HEAD"); err != nil { t.Errorf("InitRepo: %v", err) } diff --git a/clients/mockclients/repo_client.go b/clients/mockclients/repo_client.go index f6a70371ffe..8f458bcb958 100644 --- a/clients/mockclients/repo_client.go +++ b/clients/mockclients/repo_client.go @@ -94,17 +94,17 @@ func (mr *MockRepoClientMockRecorder) GetFileContent(filename interface{}) *gomo } // InitRepo mocks base method. -func (m *MockRepoClient) InitRepo(repo clients.Repo) error { +func (m *MockRepoClient) InitRepo(repo clients.Repo, commitSHA string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InitRepo", repo) + ret := m.ctrl.Call(m, "InitRepo", repo, commitSHA) ret0, _ := ret[0].(error) return ret0 } // InitRepo indicates an expected call of InitRepo. -func (mr *MockRepoClientMockRecorder) InitRepo(repo interface{}) *gomock.Call { +func (mr *MockRepoClientMockRecorder) InitRepo(repo, commitSHA interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitRepo", reflect.TypeOf((*MockRepoClient)(nil).InitRepo), repo) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitRepo", reflect.TypeOf((*MockRepoClient)(nil).InitRepo), repo, commitSHA) } // IsArchived mocks base method. diff --git a/clients/repo_client.go b/clients/repo_client.go index 5f78f46b017..71eeec343bf 100644 --- a/clients/repo_client.go +++ b/clients/repo_client.go @@ -22,7 +22,7 @@ var ErrUnsupportedFeature = errors.New("unsupported feature") // RepoClient interface is used by Scorecard checks to access a repo. type RepoClient interface { - InitRepo(repo Repo) error + InitRepo(repo Repo, commitSHA string) error URI() string IsArchived() (bool, error) ListFiles(predicate func(string) (bool, error)) ([]string, error) diff --git a/cmd/flags.go b/cmd/flags.go index 15b822e514b..965e3006ce8 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -18,6 +18,7 @@ package cmd var ( flagRepo string flagLocal string + flagCommit string flagChecksToRun []string flagMetadata []string flagLogLevel string diff --git a/cmd/root.go b/cmd/root.go index 0896e273e23..9da8acdab77 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -63,6 +63,7 @@ var rootCmd = &cobra.Command{ func init() { rootCmd.Flags().StringVar(&flagRepo, "repo", "", "repository to check") rootCmd.Flags().StringVar(&flagLocal, "local", "", "local folder to check") + rootCmd.Flags().StringVar(&flagCommit, "commit", "HEAD", "commit to analyze") rootCmd.Flags().StringVar( &flagLogLevel, "verbosity", @@ -147,6 +148,9 @@ func scorecardCmd(cmd *cobra.Command, args []string) { if flagLocal != "" { requiredRequestTypes = append(requiredRequestTypes, checker.FileBased) } + if !strings.EqualFold(flagCommit, "HEAD") { + requiredRequestTypes = append(requiredRequestTypes, checker.CommitBased) + } enabledChecks, err := getEnabledChecks(policy, flagChecksToRun, requiredRequestTypes) if err != nil { log.Panic(err) @@ -158,7 +162,7 @@ func scorecardCmd(cmd *cobra.Command, args []string) { } } - repoResult, err := pkg.RunScorecards(ctx, repoURI, flagFormat == formatRaw, enabledChecks, repoClient, + repoResult, err := pkg.RunScorecards(ctx, repoURI, flagCommit, flagFormat == formatRaw, enabledChecks, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient) if err != nil { log.Panic(err) @@ -221,12 +225,20 @@ func validateCmdFlags() { if flagFormat == formatRaw { log.Panic("raw option not supported yet") } + if flagCommit != "HEAD" { + log.Panic("--commit option not supported yet") + } } // Validate format. if !validateFormat(flagFormat) { log.Panicf("unsupported format '%s'", flagFormat) } + + // Validate `commit` is non-empty. + if flagCommit == "" { + log.Panic("commit should be non-empty") + } } func boolSum(bools ...bool) int { diff --git a/cmd/serve.go b/cmd/serve.go index fbd822fbd03..ffb6b7f263a 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -70,7 +70,7 @@ var serveCmd = &cobra.Command{ } defer ossFuzzRepoClient.Close() ciiClient := clients.DefaultCIIBestPracticesClient() - repoResult, err := pkg.RunScorecards(ctx, repo, false, checks.AllChecks, repoClient, + repoResult, err := pkg.RunScorecards(ctx, repo, "HEAD" /*commitSHA*/, false /*raw*/, checks.AllChecks, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient) if err != nil { logger.Error(err, "running enabled scorecard checks on repo") diff --git a/cron/worker/main.go b/cron/worker/main.go index 6c025d551a8..eefefae096f 100644 --- a/cron/worker/main.go +++ b/cron/worker/main.go @@ -85,7 +85,7 @@ func processRequest(ctx context.Context, continue } repo.AppendMetadata(repo.Metadata()...) - result, err := pkg.RunScorecards(ctx, repo, false, checksToRun, + result, err := pkg.RunScorecards(ctx, repo, "HEAD" /*commitSHA*/, false /*raw*/, checksToRun, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient) if errors.Is(err, sce.ErrRepoUnreachable) { // Not accessible repo - continue. diff --git a/e2e/binary_artifacts_test.go b/e2e/binary_artifacts_test.go index 35bc3903208..3c368b4d73f 100644 --- a/e2e/binary_artifacts_test.go +++ b/e2e/binary_artifacts_test.go @@ -35,7 +35,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { repo, err := githubrepo.MakeGithubRepo("ossf/scorecard") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -61,7 +61,38 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-binary-artifacts-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") + Expect(err).Should(BeNil()) + + req := checker.CheckRequest{ + Ctx: context.Background(), + RepoClient: repoClient, + Repo: repo, + Dlogger: &dl, + } + // TODO: upload real binaries to the repo as well. + expected := scut.TestReturn{ + Error: nil, + Score: checker.MinResultScore, + NumberOfWarn: 24, + NumberOfInfo: 0, + NumberOfDebug: 0, + } + result := checks.BinaryArtifacts(&req) + // UPGRADEv2: to remove. + // Old version. + Expect(result.Error).Should(BeNil()) + Expect(result.Pass).Should(BeFalse()) + // New version. + Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue()) + Expect(repoClient.Close()).Should(BeNil()) + }) + It("Should return binary artifacts present at commit in source code", func() { + dl := scut.TestDetailLogger{} + repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-binary-artifacts-e2e") + Expect(err).Should(BeNil()) + repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) + err = repoClient.InitRepo(repo, "0c6e8991781bba24bfaaa0525f69329f672ef7b5") Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -92,7 +123,34 @@ var _ = Describe("E2E TEST:"+checks.CheckBinaryArtifacts, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-binary-artifacts-e2e-4-binaries") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") + Expect(err).Should(BeNil()) + + req := checker.CheckRequest{ + Ctx: context.Background(), + RepoClient: repoClient, + Repo: repo, + Dlogger: &dl, + } + // TODO: upload real binaries to the repo as well. + expected := scut.TestReturn{ + Error: nil, + Score: checker.MaxResultScore - 4, + NumberOfWarn: 4, + NumberOfInfo: 0, + NumberOfDebug: 0, + } + result := checks.BinaryArtifacts(&req) + // New version. + Expect(scut.ValidateTestReturn(nil, "binary artifacts", &expected, &result, &dl)).Should(BeTrue()) + Expect(repoClient.Close()).Should(BeNil()) + }) + It("Should return binary artifacts present at commit in source code", func() { + dl := scut.TestDetailLogger{} + repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-binary-artifacts-e2e-4-binaries") + Expect(err).Should(BeNil()) + repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) + err = repoClient.InitRepo(repo, "d994b3e1a8912283f9958a7c1e0aa480ca24a7ce") Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/branch_protection_test.go b/e2e/branch_protection_test.go index bffaef414b4..1c04c268526 100644 --- a/e2e/branch_protection_test.go +++ b/e2e/branch_protection_test.go @@ -33,7 +33,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -63,7 +63,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e-none") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), @@ -93,7 +93,7 @@ var _ = Describe("E2E TEST:"+checks.CheckBranchProtection, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-branch-protection-e2e-patch-1") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/ci_tests_test.go b/e2e/ci_tests_test.go index a82ccf6a5e7..1eb8f5f1c3d 100644 --- a/e2e/ci_tests_test.go +++ b/e2e/ci_tests_test.go @@ -33,7 +33,31 @@ var _ = Describe("E2E TEST:"+checks.CheckCITests, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") + Expect(err).Should(BeNil()) + req := checker.CheckRequest{ + Ctx: context.Background(), + RepoClient: repoClient, + Repo: repo, + Dlogger: &dl, + } + expected := scut.TestReturn{ + Error: nil, + Score: checker.InconclusiveResultScore, + NumberOfWarn: 0, + NumberOfInfo: 0, + NumberOfDebug: 0, + } + result := checks.CITests(&req) + Expect(scut.ValidateTestReturn(nil, "CI tests run", &expected, &result, &dl)).Should(BeTrue()) + Expect(repoClient.Close()).Should(BeNil()) + }) + It("Should return use of CI tests at commit", func() { + dl := scut.TestDetailLogger{} + repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") + Expect(err).Should(BeNil()) + repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) + err = repoClient.InitRepo(repo, "0a6850647e531b08f68118ff8ca20577a5b4062c") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/code_review_test.go b/e2e/code_review_test.go index d1b654f40fd..aa7fe63622b 100644 --- a/e2e/code_review_test.go +++ b/e2e/code_review_test.go @@ -35,7 +35,32 @@ var _ = Describe("E2E TEST:CodeReview", func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") + Expect(err).Should(BeNil()) + + req := checker.CheckRequest{ + Ctx: context.Background(), + RepoClient: repoClient, + Repo: repo, + Dlogger: &dl, + } + expected := scut.TestReturn{ + Error: nil, + Score: checker.MinResultScore, + NumberOfWarn: 3, + NumberOfInfo: 0, + NumberOfDebug: 0, + } + result := checks.CodeReview(&req) + Expect(scut.ValidateTestReturn(nil, "use code reviews", &expected, &result, &dl)).Should(BeTrue()) + Expect(repoClient.Close()).Should(BeNil()) + }) + It("Should return use of code reviews at commit", func() { + dl := scut.TestDetailLogger{} + repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") + Expect(err).Should(BeNil()) + repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) + err = repoClient.InitRepo(repo, "0a6850647e531b08f68118ff8ca20577a5b4062c") Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/contributors_test.go b/e2e/contributors_test.go index c6b1b712f96..3db16d310e4 100644 --- a/e2e/contributors_test.go +++ b/e2e/contributors_test.go @@ -33,7 +33,7 @@ var _ = Describe("E2E TEST:"+checks.CheckContributors, func() { repo, err := githubrepo.MakeGithubRepo("ossf/scorecard") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/dangerous_workflow_test.go b/e2e/dangerous_workflow_test.go index eee7df6c215..668d991278a 100644 --- a/e2e/dangerous_workflow_test.go +++ b/e2e/dangerous_workflow_test.go @@ -32,7 +32,36 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-dangerous-workflow-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") + Expect(err).Should(BeNil()) + req := checker.CheckRequest{ + Ctx: context.Background(), + RepoClient: repoClient, + Repo: repo, + Dlogger: &dl, + } + expected := scut.TestReturn{ + Error: nil, + Score: checker.MinResultScore, + NumberOfWarn: 1, + NumberOfInfo: 0, + NumberOfDebug: 0, + } + result := checks.DangerousWorkflow(&req) + // UPGRADEv2: to remove. + // Old version. + + Expect(result.Error).Should(BeNil()) + Expect(result.Pass).Should(BeFalse()) + // New version. + Expect(scut.ValidateTestReturn(nil, "dangerous workflow", &expected, &result, &dl)).Should(BeTrue()) + }) + It("Should return dangerous workflow at commit", func() { + dl := scut.TestDetailLogger{} + repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-dangerous-workflow-e2e") + Expect(err).Should(BeNil()) + repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) + err = repoClient.InitRepo(repo, "8db326e9ba20517feeefd157524a89184ed41f7f") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/dependency_update_tool_test.go b/e2e/dependency_update_tool_test.go index b77df5f9b0c..928fc7764d0 100644 --- a/e2e/dependency_update_tool_test.go +++ b/e2e/dependency_update_tool_test.go @@ -35,7 +35,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() { repo, err := githubrepo.MakeGithubRepo("ossf/scorecard") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -66,7 +66,7 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() { repo, err := githubrepo.MakeGithubRepo("netlify/netlify-cms") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/fuzzing_test.go b/e2e/fuzzing_test.go index a8342250215..8f0f30c30e3 100644 --- a/e2e/fuzzing_test.go +++ b/e2e/fuzzing_test.go @@ -33,7 +33,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { repo, err := githubrepo.MakeGithubRepo("tensorflow/tensorflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) ossFuzzRepoClient, err := githubrepo.CreateOssFuzzRepoClient(context.Background(), logger) Expect(err).Should(BeNil()) @@ -61,7 +61,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-fuzzing-cflite") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) ossFuzzRepoClient, err := githubrepo.CreateOssFuzzRepoClient(context.Background(), logger) Expect(err).Should(BeNil()) @@ -89,7 +89,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-packaging-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) ossFuzzRepoClient, err := githubrepo.CreateOssFuzzRepoClient(context.Background(), logger) Expect(err).Should(BeNil()) diff --git a/e2e/license_test.go b/e2e/license_test.go index 7308ec478b6..59fdc02f3dd 100644 --- a/e2e/license_test.go +++ b/e2e/license_test.go @@ -32,7 +32,7 @@ var _ = Describe("E2E TEST:"+checks.CheckLicense, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-license-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/maintained_test.go b/e2e/maintained_test.go index e45c3a8e637..44b7a18767e 100644 --- a/e2e/maintained_test.go +++ b/e2e/maintained_test.go @@ -33,7 +33,7 @@ var _ = Describe("E2E TEST:"+checks.CheckMaintained, func() { repo, err := githubrepo.MakeGithubRepo("apache/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/packaging_test.go b/e2e/packaging_test.go index d64f6cd9d4a..f3b9d576a5a 100644 --- a/e2e/packaging_test.go +++ b/e2e/packaging_test.go @@ -33,7 +33,7 @@ var _ = Describe("E2E TEST:"+checks.CheckPackaging, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-packaging-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/permissions_test.go b/e2e/permissions_test.go index 19f0510aa08..6d0ec28566c 100644 --- a/e2e/permissions_test.go +++ b/e2e/permissions_test.go @@ -32,7 +32,37 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-token-permissions-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") + Expect(err).Should(BeNil()) + req := checker.CheckRequest{ + Ctx: context.Background(), + RepoClient: repoClient, + Repo: repo, + Dlogger: &dl, + } + expected := scut.TestReturn{ + Error: nil, + Score: checker.MinResultScore, + NumberOfWarn: 1, + NumberOfInfo: 2, + NumberOfDebug: 5, + } + result := checks.TokenPermissions(&req) + // UPGRADEv2: to remove. + // Old version. + + Expect(result.Error).Should(BeNil()) + Expect(result.Pass).Should(BeFalse()) + // New version. + Expect(scut.ValidateTestReturn(nil, "token permissions", &expected, &result, &dl)).Should(BeTrue()) + Expect(repoClient.Close()).Should(BeNil()) + }) + It("Should return token permission at commit", func() { + dl := scut.TestDetailLogger{} + repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-token-permissions-e2e") + Expect(err).Should(BeNil()) + repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) + err = repoClient.InitRepo(repo, "35a3425d1e682c32946b7d36adcfd772cf772e63") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/pinned_dependencies_test.go b/e2e/pinned_dependencies_test.go index b59f8f155f4..fd2582bdba4 100644 --- a/e2e/pinned_dependencies_test.go +++ b/e2e/pinned_dependencies_test.go @@ -34,7 +34,32 @@ var _ = Describe("E2E TEST:"+checks.CheckPinnedDependencies, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-pinned-dependencies-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") + Expect(err).Should(BeNil()) + + req := checker.CheckRequest{ + Ctx: context.Background(), + RepoClient: repoClient, + Repo: repo, + Dlogger: &dl, + } + expected := scut.TestReturn{ + Error: nil, + Score: 3, + NumberOfWarn: 139, + NumberOfInfo: 2, + NumberOfDebug: 0, + } + result := checks.PinnedDependencies(&req) + Expect(scut.ValidateTestReturn(nil, "dependencies check", &expected, &result, &dl)).Should(BeTrue()) + Expect(repoClient.Close()).Should(BeNil()) + }) + It("Should return dependencies check at commit", func() { + dl := scut.TestDetailLogger{} + repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-pinned-dependencies-e2e") + Expect(err).Should(BeNil()) + repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) + err = repoClient.InitRepo(repo, "c8bfd7cf04ea7af741e1d07af98fabfcc1b6ffb1") Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/sast_test.go b/e2e/sast_test.go index 25017f750dc..1df8b995781 100644 --- a/e2e/sast_test.go +++ b/e2e/sast_test.go @@ -33,7 +33,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSAST, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/airflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/security_policy_test.go b/e2e/security_policy_test.go index be287864177..f1c3b450a5c 100644 --- a/e2e/security_policy_test.go +++ b/e2e/security_policy_test.go @@ -32,7 +32,7 @@ var _ = Describe("E2E TEST:SecurityPolicy", func() { repo, err := githubrepo.MakeGithubRepo("tensorflow/tensorflow") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ @@ -62,7 +62,7 @@ var _ = Describe("E2E TEST:SecurityPolicy", func() { repo, err := githubrepo.MakeGithubRepo("randombit/botan") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ diff --git a/e2e/signedreleases_test.go b/e2e/signedreleases_test.go index f913c0e3a1b..28f7547ae38 100644 --- a/e2e/signedreleases_test.go +++ b/e2e/signedreleases_test.go @@ -33,7 +33,7 @@ var _ = Describe("E2E TEST:"+checks.CheckSignedReleases, func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-signed-releases-e2e") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) req := checker.CheckRequest{ Ctx: context.Background(), diff --git a/e2e/vulnerabilities_test.go b/e2e/vulnerabilities_test.go index 42c1776afa2..1e9a23b1c0d 100644 --- a/e2e/vulnerabilities_test.go +++ b/e2e/vulnerabilities_test.go @@ -33,7 +33,7 @@ var _ = Describe("E2E TEST:Vulnerabilities", func() { repo, err := githubrepo.MakeGithubRepo("ossf/scorecard") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") Expect(err).Should(BeNil()) dl := scut.TestDetailLogger{} @@ -66,7 +66,38 @@ var _ = Describe("E2E TEST:Vulnerabilities", func() { repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-vulnerabilities-open62541") Expect(err).Should(BeNil()) repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) - err = repoClient.InitRepo(repo) + err = repoClient.InitRepo(repo, "HEAD") + Expect(err).Should(BeNil()) + + dl := scut.TestDetailLogger{} + checkRequest := checker.CheckRequest{ + Ctx: context.Background(), + RepoClient: repoClient, + VulnerabilitiesClient: clients.DefaultVulnerabilitiesClient(), + Repo: repo, + Dlogger: &dl, + } + expected := scut.TestReturn{ + Error: nil, + Score: checker.MaxResultScore - 3, // 3 vulnerabilities remove 3 points. + NumberOfWarn: 1, + NumberOfInfo: 0, + NumberOfDebug: 0, + } + result := checks.Vulnerabilities(&checkRequest) + // UPGRADEv2: to remove. + // Old version. + Expect(result.Error).Should(BeNil()) + Expect(result.Pass).Should(BeFalse()) + // New version. + Expect(scut.ValidateTestReturn(nil, "osv vulnerabilities", &expected, &result, &dl)).Should(BeTrue()) + Expect(repoClient.Close()).Should(BeNil()) + }) + It("Should return that there are vulnerabilities at commit", func() { + repo, err := githubrepo.MakeGithubRepo("ossf-tests/scorecard-check-vulnerabilities-open62541") + Expect(err).Should(BeNil()) + repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger) + err = repoClient.InitRepo(repo, "de6367caa31b59e2156f83b04c2f30611b7ac393") Expect(err).Should(BeNil()) dl := scut.TestDetailLogger{} diff --git a/pkg/scorecard.go b/pkg/scorecard.go index d830bb5e096..42e2decbf0c 100644 --- a/pkg/scorecard.go +++ b/pkg/scorecard.go @@ -76,13 +76,14 @@ func getRepoCommitHash(r clients.RepoClient) (string, error) { // RunScorecards runs enabled Scorecard checks on a Repo. func RunScorecards(ctx context.Context, repo clients.Repo, + commitSHA string, raw bool, checksToRun checker.CheckNameToFnMap, repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient, ciiClient clients.CIIBestPracticesClient, vulnsClient clients.VulnerabilitiesClient) (ScorecardResult, error) { - if err := repoClient.InitRepo(repo); err != nil { + if err := repoClient.InitRepo(repo, commitSHA); err != nil { // No need to call sce.WithMessage() since InitRepo will do that for us. //nolint:wrapcheck return ScorecardResult{}, err