diff --git a/cron/controller/main.go b/cron/controller/main.go index a11ea53aec7..dd43fa70889 100644 --- a/cron/controller/main.go +++ b/cron/controller/main.go @@ -24,12 +24,15 @@ import ( "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/ossf/scorecard/v4/clients" "github.com/ossf/scorecard/v4/cron/config" "github.com/ossf/scorecard/v4/cron/data" "github.com/ossf/scorecard/v4/cron/pubsub" "github.com/ossf/scorecard/v4/pkg" ) +var headSHA = clients.HeadSHA + func publishToRepoRequestTopic(iter data.Iterator, topicPublisher pubsub.Publisher, shardSize int, datetime time.Time) (int32, error) { var shardNum int32 @@ -48,7 +51,9 @@ func publishToRepoRequestTopic(iter data.Iterator, topicPublisher pubsub.Publish return shardNum, fmt.Errorf("error reading repoURL: %w", err) } request.Repos = append(request.GetRepos(), &data.Repo{ - Url: &repoURL.Repo, + Url: &repoURL.Repo, + // TODO(controller): pass in non-HEAD commitSHA here. + Commit: &headSHA, Metadata: repoURL.Metadata.ToString(), }) if len(request.GetRepos()) < shardSize { diff --git a/cron/data/request.pb.go b/cron/data/request.pb.go index 14ac07cdb1c..b219e0642d1 100644 --- a/cron/data/request.pb.go +++ b/cron/data/request.pb.go @@ -21,12 +21,11 @@ package data import ( - reflect "reflect" - sync "sync" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" ) const ( @@ -42,6 +41,7 @@ type Repo struct { unknownFields protoimpl.UnknownFields Url *string `protobuf:"bytes,1,opt,name=url,proto3,oneof" json:"url,omitempty"` + Commit *string `protobuf:"bytes,3,opt,name=commit,proto3,oneof" json:"commit,omitempty"` Metadata []string `protobuf:"bytes,2,rep,name=metadata,proto3" json:"metadata,omitempty"` } @@ -84,6 +84,13 @@ func (x *Repo) GetUrl() string { return "" } +func (x *Repo) GetCommit() string { + if x != nil && x.Commit != nil { + return *x.Commit + } + return "" +} + func (x *Repo) GetMetadata() []string { if x != nil { return x.Metadata @@ -162,27 +169,29 @@ var file_cron_data_request_proto_rawDesc = []byte{ 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2e, 0x63, 0x72, 0x6f, 0x6e, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x41, 0x0a, 0x04, 0x52, 0x65, 0x70, 0x6f, 0x12, 0x15, 0x0a, 0x03, + 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x69, 0x0a, 0x04, 0x52, 0x65, 0x70, 0x6f, 0x12, 0x15, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x03, 0x75, 0x72, 0x6c, - 0x88, 0x01, 0x01, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x42, - 0x06, 0x0a, 0x04, 0x5f, 0x75, 0x72, 0x6c, 0x22, 0xcc, 0x01, 0x0a, 0x15, 0x53, 0x63, 0x6f, 0x72, - 0x65, 0x63, 0x61, 0x72, 0x64, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x34, 0x0a, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1e, 0x2e, 0x6f, 0x73, 0x73, 0x66, 0x2e, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, - 0x64, 0x2e, 0x63, 0x72, 0x6f, 0x6e, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x65, 0x70, 0x6f, - 0x52, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x20, 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, - 0x5f, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x08, 0x73, 0x68, - 0x61, 0x72, 0x64, 0x4e, 0x75, 0x6d, 0x88, 0x01, 0x01, 0x12, 0x3a, 0x0a, 0x08, 0x6a, 0x6f, 0x62, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x01, 0x52, 0x07, 0x6a, 0x6f, 0x62, 0x54, 0x69, - 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, - 0x6e, 0x75, 0x6d, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x6a, 0x6f, 0x62, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x42, 0x25, 0x5a, 0x23, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x73, 0x73, 0x66, 0x2f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, - 0x61, 0x72, 0x64, 0x2f, 0x63, 0x72, 0x6f, 0x6e, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x88, 0x01, 0x01, + 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x42, 0x06, 0x0a, 0x04, + 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x22, + 0xcc, 0x01, 0x0a, 0x15, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x42, 0x61, 0x74, + 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x05, 0x72, 0x65, 0x70, + 0x6f, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x73, 0x73, 0x66, 0x2e, + 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2e, 0x63, 0x72, 0x6f, 0x6e, 0x2e, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x52, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x12, + 0x20, 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x05, 0x48, 0x00, 0x52, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x4e, 0x75, 0x6d, 0x88, 0x01, + 0x01, 0x12, 0x3a, 0x0a, 0x08, 0x6a, 0x6f, 0x62, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, + 0x01, 0x52, 0x07, 0x6a, 0x6f, 0x62, 0x54, 0x69, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, + 0x0a, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x42, 0x0b, 0x0a, 0x09, 0x5f, + 0x6a, 0x6f, 0x62, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x42, 0x25, + 0x5a, 0x23, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x73, 0x73, + 0x66, 0x2f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2f, 0x63, 0x72, 0x6f, 0x6e, + 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/cron/data/request.proto b/cron/data/request.proto index dab6d3e9797..423e21e0b35 100644 --- a/cron/data/request.proto +++ b/cron/data/request.proto @@ -22,6 +22,7 @@ option go_package = "github.com/ossf/scorecard/cron/data"; message Repo { optional string url = 1; + optional string commit = 3; repeated string metadata = 2; } diff --git a/cron/worker/main.go b/cron/worker/main.go index e9c3b0c17ba..b0eafd425db 100644 --- a/cron/worker/main.go +++ b/cron/worker/main.go @@ -27,7 +27,6 @@ import ( "go.opencensus.io/stats/view" "github.com/ossf/scorecard/v4/checker" - "github.com/ossf/scorecard/v4/checks" "github.com/ossf/scorecard/v4/clients" "github.com/ossf/scorecard/v4/clients/githubrepo" githubstats "github.com/ossf/scorecard/v4/clients/githubrepo/stats" @@ -40,14 +39,17 @@ import ( sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/log" "github.com/ossf/scorecard/v4/pkg" + "github.com/ossf/scorecard/v4/policy" "github.com/ossf/scorecard/v4/stats" ) var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to true any runtime errors will be ignored") +// nolint: gocognit func processRequest(ctx context.Context, - batchRequest *data.ScorecardBatchRequest, checksToRun checker.CheckNameToFnMap, - bucketURL, bucketURL2 string, checkDocs docs.Doc, + batchRequest *data.ScorecardBatchRequest, + blacklistedChecks []string, bucketURL, bucketURL2 string, + checkDocs docs.Doc, repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient, ciiClient clients.CIIBestPracticesClient, vulnsClient clients.VulnerabilitiesClient, @@ -74,16 +76,30 @@ func processRequest(ctx context.Context, var buffer bytes.Buffer var buffer2 bytes.Buffer // TODO: run Scorecard for each repo in a separate thread. - for _, repo := range batchRequest.GetRepos() { - logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", *repo.Url)) - repo, err := githubrepo.MakeGithubRepo(*repo.Url) + for _, repoReq := range batchRequest.GetRepos() { + logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", *repoReq.Url)) + repo, err := githubrepo.MakeGithubRepo(*repoReq.Url) if err != nil { // TODO(log): Previously Warn. Consider logging an error here. logger.Info(fmt.Sprintf("invalid GitHub URL: %v", err)) continue } repo.AppendMetadata(repo.Metadata()...) - result, err := pkg.RunScorecards(ctx, repo, clients.HeadSHA /*commitSHA*/, false /*raw*/, checksToRun, + + commitSHA := clients.HeadSHA + requiredRequestType := []checker.RequestType{} + if repoReq.Commit != nil && *repoReq.Commit != clients.HeadSHA { + commitSHA = *repoReq.Commit + requiredRequestType = append(requiredRequestType, checker.CommitBased) + } + checksToRun, err := policy.GetEnabled(nil /*policy*/, nil /*checks*/, requiredRequestType) + if err != nil { + return fmt.Errorf("error during policy.GetEnabled: %w", err) + } + for _, check := range blacklistedChecks { + delete(checksToRun, check) + } + result, err := pkg.RunScorecards(ctx, repo, commitSHA, false /*raw*/, checksToRun, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient) if errors.Is(err, sce.ErrRepoUnreachable) { // Not accessible repo - continue. @@ -207,10 +223,6 @@ func main() { logger.Info(fmt.Sprintf("%v", http.ListenAndServe(":8080", nil))) }() - checksToRun := checks.AllChecks - for _, check := range blacklistedChecks { - delete(checksToRun, check) - } for { req, err := subscriber.SynchronousPull() if err != nil { @@ -223,7 +235,7 @@ func main() { logger.Info("subscription returned nil message during Receive, exiting") break } - if err := processRequest(ctx, req, checksToRun, + if err := processRequest(ctx, req, blacklistedChecks, bucketURL, bucketURL2, checkDocs, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil { // TODO(log): Previously Warn. Consider logging an error here.