✨ Add raw results to cron scans (#1741)

* draft * updates * updates * updates * updates * updates * comments * comments * comments * comments * comments * comments
ossf · Mar 19, 2022 · b1ab16e · b1ab16e
1 parent d5893c2
commit b1ab16e
Show file tree

Hide file tree

Showing 19 changed files with 481 additions and 29 deletions.
diff --git a/checks/binary_artifact.go b/checks/binary_artifact.go
@@ -26,7 +26,7 @@ const CheckBinaryArtifacts string = "Binary-Artifacts"
 
 //nolint
 func init() {
-	var supportedRequestTypes = []checker.RequestType{
+	supportedRequestTypes := []checker.RequestType{
 		checker.FileBased,
 		checker.CommitBased,
 	}
@@ -47,7 +47,6 @@ func BinaryArtifacts(c *checker.CheckRequest) checker.CheckResult {
 	// Return raw results.
 	if c.RawResults != nil {
 		c.RawResults.BinaryArtifactResults = rawData
-		return checker.CheckResult{}
 	}
 
 	// Return the score evaluation.

diff --git a/checks/branch_protection.go b/checks/branch_protection.go
@@ -43,7 +43,6 @@ func BranchProtection(c *checker.CheckRequest) checker.CheckResult {
 	// Return raw results.
 	if c.RawResults != nil {
 		c.RawResults.BranchProtectionResults = rawData
-		return checker.CheckResult{}
 	}
 
 	// Return the score evaluation.

diff --git a/checks/code_review.go b/checks/code_review.go
@@ -46,7 +46,6 @@ func CodeReview(c *checker.CheckRequest) checker.CheckResult {
 	// Return raw results.
 	if c.RawResults != nil {
 		c.RawResults.CodeReviewResults = rawData
-		return checker.CheckResult{}
 	}
 
 	// Return the score evaluation.

diff --git a/checks/dependency_update_tool.go b/checks/dependency_update_tool.go
@@ -26,7 +26,7 @@ const CheckDependencyUpdateTool = "Dependency-Update-Tool"
 
 //nolint
 func init() {
-	var supportedRequestTypes = []checker.RequestType{
+	supportedRequestTypes := []checker.RequestType{
 		checker.FileBased,
 	}
 	if err := registerCheck(CheckDependencyUpdateTool, DependencyUpdateTool, supportedRequestTypes); err != nil {
@@ -46,7 +46,6 @@ func DependencyUpdateTool(c *checker.CheckRequest) checker.CheckResult {
 	// Return raw results.
 	if c.RawResults != nil {
 		c.RawResults.DependencyUpdateToolResults = rawData
-		return checker.CheckResult{}
 	}
 
 	// Return the score evaluation.

diff --git a/checks/security_policy.go b/checks/security_policy.go
@@ -47,7 +47,6 @@ func SecurityPolicy(c *checker.CheckRequest) checker.CheckResult {
 	// Set the raw results.
 	if c.RawResults != nil {
 		c.RawResults.SecurityPolicyResults = rawData
-		return checker.CheckResult{}
 	}
 
 	return evaluation.SecurityPolicy(CheckSecurityPolicy, c.Dlogger, &rawData)

diff --git a/checks/vulnerabilities.go b/checks/vulnerabilities.go
@@ -46,7 +46,6 @@ func Vulnerabilities(c *checker.CheckRequest) checker.CheckResult {
 	// Set the raw results.
 	if c.RawResults != nil {
 		c.RawResults.VulnerabilitiesResults = rawData
-		return checker.CheckResult{}
 	}
 
 	return evaluation.Vulnerabilities(CheckVulnerabilities, c.Dlogger, &rawData)

diff --git a/cmd/root.go b/cmd/root.go
@@ -128,7 +128,6 @@ func rootCmd(o *options.Options) {
 		ctx,
 		repoURI,
 		o.Commit,
-		o.Format == options.FormatRaw,
 		enabledChecks,
 		repoClient,
 		ossFuzzRepoClient,

diff --git a/cmd/serve.go b/cmd/serve.go
@@ -69,7 +69,7 @@ func serveCmd(o *options.Options) *cobra.Command {
 				defer ossFuzzRepoClient.Close()
 				ciiClient := clients.DefaultCIIBestPracticesClient()
 				repoResult, err := pkg.RunScorecards(
-					ctx, repo, clients.HeadSHA /*commitSHA*/, false /*raw*/, checks.AllChecks, repoClient,
+					ctx, repo, clients.HeadSHA /*commitSHA*/, checks.AllChecks, repoClient,
 					ossFuzzRepoClient, ciiClient, vulnsClient)
 				if err != nil {
 					logger.Error(err, "running enabled scorecard checks on repo")

diff --git a/cron/config/config.go b/cron/config/config.go
@@ -47,9 +47,12 @@ const (
 	metricExporter         string = "SCORECARD_METRIC_EXPORTER"
 	ciiDataBucketURL       string = "SCORECARD_CII_DATA_BUCKET_URL"
 	blacklistedChecks      string = "SCORECARD_BLACKLISTED_CHECKS"
-
+	// v2 results.
 	bigqueryTableV2       string = "SCORECARD_BIGQUERY_TABLEV2"
 	resultDataBucketURLV2 string = "SCORECARD_DATA_BUCKET_URLV2"
+	// Raw results.
+	rawBigqueryTable       string = "RAW_SCORECARD_BIGQUERY_TABLE"
+	rawResultDataBucketURL string = "RAW_SCORECARD_DATA_BUCKET_URL"
 )
 
 var (
@@ -78,6 +81,9 @@ type config struct {
 	// UPGRADEv2: to remove.
 	ResultDataBucketURLV2 string `yaml:"result-data-bucket-url-v2"`
 	BigQueryTableV2       string `yaml:"bigquery-table-v2"`
+	// Raw results.
+	RawResultDataBucketURL string `yaml:"raw-result-data-bucket-url"`
+	RawBigQueryTable       string `yaml:"raw-bigquery-table"`
 }
 
 func getParsedConfigFromFile(byteValue []byte) (config, error) {
@@ -196,6 +202,18 @@ func GetResultDataBucketURLV2() (string, error) {
 	return getStringConfigValue(resultDataBucketURLV2, configYAML, "ResultDataBucketURLV2", "result-data-bucket-url-v2")
 }
 
+// GetRawBigQueryTable returns the table name to transfer cron job results.
+func GetRawBigQueryTable() (string, error) {
+	return getStringConfigValue(rawBigqueryTable, configYAML,
+		"RawBigQueryTable", "raw-bigquery-table")
+}
+
+// GetRawResultDataBucketURL returns the bucketURL for storing cron job's raw results.
+func GetRawResultDataBucketURL() (string, error) {
+	return getStringConfigValue(rawResultDataBucketURL, configYAML,
+		"RawResultDataBucketURL", "raw-result-data-bucket-url")
+}
+
 // GetShardSize returns the shard_size for the cron job.
 func GetShardSize() (int, error) {
 	return getIntConfigValue(shardSize, configYAML, "ShardSize", "shard-size")

diff --git a/cron/config/config.yaml b/cron/config/config.yaml
@@ -29,3 +29,6 @@ metric-exporter: stackdriver
 # UPGRADEv2: to remove.
 result-data-bucket-url-v2: gs://ossf-scorecard-data2
 bigquery-table-v2: scorecard-v2
+# Raw results.
+raw-result-data-bucket-url: gs://ossf-scorecard-rawdata
+raw-bigquery-table: scorecard-rawdata
diff --git a/cron/config/config_test.go b/cron/config/config_test.go
@@ -39,6 +39,9 @@ const (
 	// UPGRADEv2: to remove.
 	prodBucketV2        = "gs://ossf-scorecard-data2"
 	prodBigQueryTableV2 = "scorecard-v2"
+	// Raw results.
+	prodRawBucket        = "gs://ossf-scorecard-rawdata"
+	prodRawBigQueryTable = "scorecard-rawdata"
 )
 
 func getByteValueFromFile(filename string) ([]byte, error) {
@@ -73,8 +76,10 @@ func TestYAMLParsing(t *testing.T) {
 				ShardSize:              prodShardSize,
 				MetricExporter:         prodMetricExporter,
 				// UPGRADEv2: to remove.
-				ResultDataBucketURLV2: prodBucketV2,
-				BigQueryTableV2:       prodBigQueryTableV2,
+				ResultDataBucketURLV2:  prodBucketV2,
+				BigQueryTableV2:        prodBigQueryTableV2,
+				RawResultDataBucketURL: prodRawBucket,
+				RawBigQueryTable:       prodRawBigQueryTable,
 			},
 		},
 

diff --git a/cron/controller/main.go b/cron/controller/main.go
@@ -34,7 +34,8 @@ import (
 var headSHA = clients.HeadSHA
 
 func publishToRepoRequestTopic(iter data.Iterator, topicPublisher pubsub.Publisher,
-	shardSize int, datetime time.Time) (int32, error) {
+	shardSize int, datetime time.Time,
+) (int32, error) {
 	var shardNum int32
 	request := data.ScorecardBatchRequest{
 		JobTime:  timestamppb.New(datetime),
@@ -122,6 +123,11 @@ func main() {
 		panic(err)
 	}
 
+	rawBucket, err := config.GetRawResultDataBucketURL()
+	if err != nil {
+		panic(err)
+	}
+
 	shardNum, err := publishToRepoRequestTopic(reader, topicPublisher, shardSize, t)
 	if err != nil {
 		panic(err)
@@ -154,4 +160,15 @@ func main() {
 	if err != nil {
 		panic(fmt.Errorf("error writing to BlobStore2: %w", err))
 	}
+
+	// Raw data.
+	*metadata.ShardLoc = rawBucket + "/" + data.GetBlobFilename("", t)
+	metadataJSON, err = protojson.Marshal(&metadata)
+	if err != nil {
+		panic(fmt.Errorf("error during protojson.Marshal raw: %w", err))
+	}
+	err = data.WriteToBlobStore(ctx, rawBucket, data.GetShardMetadataFilename(t), metadataJSON)
+	if err != nil {
+		panic(fmt.Errorf("error writing to BlobStore raw: %w", err))
+	}
 }