Pass in specific commit-SHA in cron job (#1739)

Co-authored-by: Azeem Shaikh <[email protected]>

cron/controller/main.go

@@ -24,12 +24,15 @@ import (
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
+	"github.com/ossf/scorecard/v4/clients"
 	"github.com/ossf/scorecard/v4/cron/config"
 	"github.com/ossf/scorecard/v4/cron/data"
 	"github.com/ossf/scorecard/v4/cron/pubsub"
 	"github.com/ossf/scorecard/v4/pkg"
 )
 
+var headSHA = clients.HeadSHA
+
 func publishToRepoRequestTopic(iter data.Iterator, topicPublisher pubsub.Publisher,
 	shardSize int, datetime time.Time) (int32, error) {
 	var shardNum int32
@@ -48,7 +51,9 @@ func publishToRepoRequestTopic(iter data.Iterator, topicPublisher pubsub.Publish
 			return shardNum, fmt.Errorf("error reading repoURL: %w", err)
 		}
 		request.Repos = append(request.GetRepos(), &data.Repo{
-			Url:      &repoURL.Repo,
+			Url: &repoURL.Repo,
+			// TODO(controller): pass in non-HEAD commitSHA here.
+			Commit:   &headSHA,
 			Metadata: repoURL.Metadata.ToString(),
 		})
 		if len(request.GetRepos()) < shardSize {

cron/data/request.proto

@@ -22,6 +22,7 @@ option go_package = "github.com/ossf/scorecard/cron/data";
 
 message Repo {
   optional string url = 1;
+  optional string commit = 3;
   repeated string metadata = 2;
 }
 

cron/worker/main.go

@@ -27,7 +27,6 @@ import (
 	"go.opencensus.io/stats/view"
 
 	"github.com/ossf/scorecard/v4/checker"
-	"github.com/ossf/scorecard/v4/checks"
 	"github.com/ossf/scorecard/v4/clients"
 	"github.com/ossf/scorecard/v4/clients/githubrepo"
 	githubstats "github.com/ossf/scorecard/v4/clients/githubrepo/stats"
@@ -40,14 +39,17 @@ import (
 	sce "github.com/ossf/scorecard/v4/errors"
 	"github.com/ossf/scorecard/v4/log"
 	"github.com/ossf/scorecard/v4/pkg"
+	"github.com/ossf/scorecard/v4/policy"
 	"github.com/ossf/scorecard/v4/stats"
 )
 
 var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to true any runtime errors will be ignored")
 
+// nolint: gocognit
 func processRequest(ctx context.Context,
-	batchRequest *data.ScorecardBatchRequest, checksToRun checker.CheckNameToFnMap,
-	bucketURL, bucketURL2 string, checkDocs docs.Doc,
+	batchRequest *data.ScorecardBatchRequest,
+	blacklistedChecks []string, bucketURL, bucketURL2 string,
+	checkDocs docs.Doc,
 	repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient,
 	ciiClient clients.CIIBestPracticesClient,
 	vulnsClient clients.VulnerabilitiesClient,
@@ -74,16 +76,30 @@ func processRequest(ctx context.Context,
 	var buffer bytes.Buffer
 	var buffer2 bytes.Buffer
 	// TODO: run Scorecard for each repo in a separate thread.
-	for _, repo := range batchRequest.GetRepos() {
-		logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", *repo.Url))
-		repo, err := githubrepo.MakeGithubRepo(*repo.Url)
+	for _, repoReq := range batchRequest.GetRepos() {
+		logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", *repoReq.Url))
+		repo, err := githubrepo.MakeGithubRepo(*repoReq.Url)
 		if err != nil {
 			// TODO(log): Previously Warn. Consider logging an error here.
 			logger.Info(fmt.Sprintf("invalid GitHub URL: %v", err))
 			continue
 		}
 		repo.AppendMetadata(repo.Metadata()...)
-		result, err := pkg.RunScorecards(ctx, repo, clients.HeadSHA /*commitSHA*/, false /*raw*/, checksToRun,
+
+		commitSHA := clients.HeadSHA
+		requiredRequestType := []checker.RequestType{}
+		if repoReq.Commit != nil && *repoReq.Commit != clients.HeadSHA {
+			commitSHA = *repoReq.Commit
+			requiredRequestType = append(requiredRequestType, checker.CommitBased)
+		}
+		checksToRun, err := policy.GetEnabled(nil /*policy*/, nil /*checks*/, requiredRequestType)
+		if err != nil {
+			return fmt.Errorf("error during policy.GetEnabled: %w", err)
+		}
+		for _, check := range blacklistedChecks {
+			delete(checksToRun, check)
+		}
+		result, err := pkg.RunScorecards(ctx, repo, commitSHA, false /*raw*/, checksToRun,
 			repoClient, ossFuzzRepoClient, ciiClient, vulnsClient)
 		if errors.Is(err, sce.ErrRepoUnreachable) {
 			// Not accessible repo - continue.
@@ -207,10 +223,6 @@ func main() {
 		logger.Info(fmt.Sprintf("%v", http.ListenAndServe(":8080", nil)))
 	}()
 
-	checksToRun := checks.AllChecks
-	for _, check := range blacklistedChecks {
-		delete(checksToRun, check)
-	}
 	for {
 		req, err := subscriber.SynchronousPull()
 		if err != nil {
@@ -223,7 +235,7 @@ func main() {
 			logger.Info("subscription returned nil message during Receive, exiting")
 			break
 		}
-		if err := processRequest(ctx, req, checksToRun,
+		if err := processRequest(ctx, req, blacklistedChecks,
 			bucketURL, bucketURL2, checkDocs,
 			repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil {
 			// TODO(log): Previously Warn. Consider logging an error here.