Run Dangerous-Workflow in release tests (#1301)

Co-authored-by: Azeem Shaikh <[email protected]>

cron/config/config.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"reflect"
 	"strconv"
+	"strings"
 
 	"gopkg.in/yaml.v2"
 )
@@ -46,6 +47,7 @@ const (
 	webhookURL             string = "SCORECARD_WEBHOOK_URL"
 	metricExporter         string = "SCORECARD_METRIC_EXPORTER"
 	ciiDataBucketURL       string = "SCORECARD_CII_DATA_BUCKET_URL"
+	blacklistedChecks      string = "SCORECARD_BLACKLISTED_CHECKS"
 
 	bigqueryTableV2       string = "SCORECARD_BIGQUERY_TABLEV2"
 	resultDataBucketURLV2 string = "SCORECARD_DATA_BUCKET_URLV2"
@@ -71,6 +73,7 @@ type config struct {
 	CompletionThreshold    float32 `yaml:"completion-threshold"`
 	WebhookURL             string  `yaml:"webhook-url"`
 	CIIDataBucketURL       string  `yaml:"cii-data-bucket-url"`
+	BlacklistedChecks      string  `yaml:"blacklisted-checks"`
 	MetricExporter         string  `yaml:"metric-exporter"`
 	ShardSize              int     `yaml:"shard-size"`
 	// UPGRADEv2: to remove.
@@ -217,6 +220,15 @@ func GetCIIDataBucketURL() (string, error) {
 	return url, nil
 }
 
+// GetBlacklistedChecks returns a list of checks which are not to be run.
+func GetBlacklistedChecks() ([]string, error) {
+	checks, err := getStringConfigValue(blacklistedChecks, configYAML, "BlacklistedChecks", "blacklisted-checks")
+	if err != nil && !errors.Is(err, ErrorEmptyConfigValue) {
+		return nil, err
+	}
+	return strings.Split(checks, ","), nil
+}
+
 // GetMetricExporter returns the opencensus exporter type.
 func GetMetricExporter() (string, error) {
 	return getStringConfigValue(metricExporter, configYAML, "MetricExporter", "metric-exporter")

cron/config/config.yaml

@@ -22,6 +22,10 @@ completion-threshold: 0.99
 shard-size: 10
 webhook-url: 
 cii-data-bucket-url: gs://ossf-scorecard-cii-data
+# TODO: Temporarily remove SAST and CI-Tests which require lot of GitHub API tokens.
+# TODO(#859): Re-add Contributors after fixing inconsistencies.
+# TODO: Add Dangerous-Workflow in v4
+blacklisted-checks: SAST,CI-Tests,Contributors,Dangerous-Workflow
 metric-exporter: stackdriver
 # UPGRADEv2: to remove.
 result-data-bucket-url-v2: gs://ossf-scorecard-data2

cron/config/config_test.go

@@ -33,6 +33,7 @@ const (
 	prodCompletionThreshold        = 0.99
 	prodWebhookURL                 = ""
 	prodCIIDataBucket              = "gs://ossf-scorecard-cii-data"
+	prodBlacklistedChecks          = "SAST,CI-Tests,Contributors,Dangerous-Workflow"
 	prodShardSize           int    = 10
 	prodMetricExporter      string = "stackdriver"
 	// UPGRADEv2: to remove.
@@ -68,6 +69,7 @@ func TestYAMLParsing(t *testing.T) {
 				CompletionThreshold:    prodCompletionThreshold,
 				WebhookURL:             prodWebhookURL,
 				CIIDataBucketURL:       prodCIIDataBucket,
+				BlacklistedChecks:      prodBlacklistedChecks,
 				ShardSize:              prodShardSize,
 				MetricExporter:         prodMetricExporter,
 				// UPGRADEv2: to remove.

cron/k8s/worker.release.yaml

@@ -39,6 +39,8 @@ spec:
           value: "gs://ossf-scorecard-data-releasetest2"
         - name: SCORECARD_REQUEST_SUBSCRIPTION_URL
           value: "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker-releasetest"
+        - name: SCORECARD_BLAKCLISTED_CHECKS
+          value: "SAST,CI-Tests,Contributors"
         - name: SCORECARD_METRIC_EXPORTER
           value: "printer"
         - name: GITHUB_AUTH_SERVER

cron/worker/main.go

@@ -173,6 +173,11 @@ func main() {
 		panic(err)
 	}
 
+	blacklistedChecks, err := config.GetBlacklistedChecks()
+	if err != nil {
+		panic(err)
+	}
+
 	ciiDataBucketURL, err := config.GetCIIDataBucketURL()
 	if err != nil {
 		panic(err)
@@ -202,13 +207,9 @@ func main() {
 	}()
 
 	checksToRun := checks.AllChecks
-	// TODO: Temporarily remove checks which require lot of GitHub API token.
-	delete(checksToRun, checks.CheckSAST)
-	delete(checksToRun, checks.CheckCITests)
-	// TODO: Re-add Contributors check after fixing: #859.
-	delete(checksToRun, checks.CheckContributors)
-	// TODO: Add this in v4
-	delete(checksToRun, checks.CheckDangerousWorkflow)
+	for _, check := range blacklistedChecks {
+		delete(checksToRun, check)
+	}
 	for {
 		req, err := subscriber.SynchronousPull()
 		if err != nil {