From c0cf0011963253d8c1c77904effef6a8d93819c1 Mon Sep 17 00:00:00 2001 From: Caleb Brown Date: Fri, 11 Nov 2022 15:43:01 +1100 Subject: [PATCH] :seedling: cron: expose the stackdriver prefix as a config variable so it can be changed. (#2446) * Expose the stackdriver prefix as a config variable so it can be changed. Signed-off-by: Caleb Brown * fix linter warning Signed-off-by: Caleb Brown Signed-off-by: Caleb Brown Co-authored-by: Spencer Schrock Signed-off-by: nathaniel.wert --- cron/config/config.go | 60 +++++++++++++----------- cron/config/config.yaml | 1 + cron/config/config_test.go | 69 +++++++++++++++++----------- cron/internal/monitoring/exporter.go | 7 ++- 4 files changed, 82 insertions(+), 55 deletions(-) diff --git a/cron/config/config.go b/cron/config/config.go index cbc47293313b..e913cef61fd9 100644 --- a/cron/config/config.go +++ b/cron/config/config.go @@ -42,19 +42,20 @@ const ( configUsage string = "Location of config file. Required" inputBucketParams string = "input-bucket" - projectID string = "SCORECARD_PROJECT_ID" - requestTopicURL string = "SCORECARD_REQUEST_TOPIC_URL" - requestSubscriptionURL string = "SCORECARD_REQUEST_SUBSCRIPTION_URL" - bigqueryDataset string = "SCORECARD_BIGQUERY_DATASET" - completionThreshold string = "SCORECARD_COMPLETION_THRESHOLD" - shardSize string = "SCORECARD_SHARD_SIZE" - webhookURL string = "SCORECARD_WEBHOOK_URL" - metricExporter string = "SCORECARD_METRIC_EXPORTER" - bigqueryTable string = "SCORECARD_BIGQUERY_TABLE" - resultDataBucketURL string = "SCORECARD_DATA_BUCKET_URL" - apiResultsBucketURL string = "SCORECARD_API_RESULTS_BUCKET_URL" - inputBucketURL string = "SCORECARD_INPUT_BUCKET_URL" - inputBucketPrefix string = "SCORECARD_INPUT_BUCKET_PREFIX" + projectID string = "SCORECARD_PROJECT_ID" + requestTopicURL string = "SCORECARD_REQUEST_TOPIC_URL" + requestSubscriptionURL string = "SCORECARD_REQUEST_SUBSCRIPTION_URL" + bigqueryDataset string = "SCORECARD_BIGQUERY_DATASET" + completionThreshold string = "SCORECARD_COMPLETION_THRESHOLD" + shardSize string = "SCORECARD_SHARD_SIZE" + webhookURL string = "SCORECARD_WEBHOOK_URL" + metricExporter string = "SCORECARD_METRIC_EXPORTER" + metricStackdriverPrefix string = "SCORECARD_METRIC_STACKDRIVER_PREFIX" + bigqueryTable string = "SCORECARD_BIGQUERY_TABLE" + resultDataBucketURL string = "SCORECARD_DATA_BUCKET_URL" + apiResultsBucketURL string = "SCORECARD_API_RESULTS_BUCKET_URL" + inputBucketURL string = "SCORECARD_INPUT_BUCKET_URL" + inputBucketPrefix string = "SCORECARD_INPUT_BUCKET_PREFIX" ) var ( @@ -71,19 +72,20 @@ var ( //nolint:govet type config struct { - ProjectID string `yaml:"project-id"` - ResultDataBucketURL string `yaml:"result-data-bucket-url"` - RequestTopicURL string `yaml:"request-topic-url"` - RequestSubscriptionURL string `yaml:"request-subscription-url"` - BigQueryDataset string `yaml:"bigquery-dataset"` - BigQueryTable string `yaml:"bigquery-table"` - CompletionThreshold float32 `yaml:"completion-threshold"` - WebhookURL string `yaml:"webhook-url"` - MetricExporter string `yaml:"metric-exporter"` - ShardSize int `yaml:"shard-size"` - InputBucketURL string `yaml:"input-bucket-url"` - InputBucketPrefix string `yaml:"input-bucket-prefix"` - AdditionalParams map[string]map[string]string `yaml:"additional-params"` + ProjectID string `yaml:"project-id"` + ResultDataBucketURL string `yaml:"result-data-bucket-url"` + RequestTopicURL string `yaml:"request-topic-url"` + RequestSubscriptionURL string `yaml:"request-subscription-url"` + BigQueryDataset string `yaml:"bigquery-dataset"` + BigQueryTable string `yaml:"bigquery-table"` + CompletionThreshold float32 `yaml:"completion-threshold"` + WebhookURL string `yaml:"webhook-url"` + MetricExporter string `yaml:"metric-exporter"` + MetricStackdriverPrefix string `yaml:"metric-stackdriver-prefix"` + ShardSize int `yaml:"shard-size"` + InputBucketURL string `yaml:"input-bucket-url"` + InputBucketPrefix string `yaml:"input-bucket-prefix"` + AdditionalParams map[string]map[string]string `yaml:"additional-params"` } func getParsedConfigFromFile(byteValue []byte) (config, error) { @@ -294,6 +296,12 @@ func GetMetricExporter() (string, error) { return getStringConfigValue(metricExporter, configYAML, "MetricExporter", "metric-exporter") } +// GetMetricStackdriverPrefix returns the prefix for stackdriver opencensus exporter. +func GetMetricStackdriverPrefix() (string, error) { + return getStringConfigValue( + metricStackdriverPrefix, configYAML, "MetricStackdriverPrefix", "metric-stackdriver-prefix") +} + // GetAPIResultsBucketURL returns the bucket URL for storing cron job results. func GetAPIResultsBucketURL() (string, error) { return getScorecardParam("api-results-bucket-url") diff --git a/cron/config/config.yaml b/cron/config/config.yaml index e9d387e8da44..7d279ca8ef33 100644 --- a/cron/config/config.yaml +++ b/cron/config/config.yaml @@ -21,6 +21,7 @@ completion-threshold: 0.99 shard-size: 10 webhook-url: metric-exporter: stackdriver +metric-stackdriver-prefix: scorecard-cron result-data-bucket-url: gs://ossf-scorecard-data2 # TODO temporarily leaving old variables until changes propagate to production diff --git a/cron/config/config_test.go b/cron/config/config_test.go index 3f9851d470b7..397fd6eb8226 100644 --- a/cron/config/config_test.go +++ b/cron/config/config_test.go @@ -24,19 +24,20 @@ import ( ) const ( - testEnvVar string = "TEST_ENV_VAR" - prodProjectID = "openssf" - prodBucket = "gs://ossf-scorecard-data2" - prodTopic = "gcppubsub://projects/openssf/topics/scorecard-batch-requests" - prodSubscription = "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker" - prodBigQueryDataset = "scorecardcron" - prodBigQueryTable = "scorecard-v2" - prodCompletionThreshold = 0.99 - prodWebhookURL = "" - prodCIIDataBucket = "gs://ossf-scorecard-cii-data" - prodBlacklistedChecks = "CI-Tests,Contributors" - prodShardSize int = 10 - prodMetricExporter string = "stackdriver" + testEnvVar string = "TEST_ENV_VAR" + prodProjectID = "openssf" + prodBucket = "gs://ossf-scorecard-data2" + prodTopic = "gcppubsub://projects/openssf/topics/scorecard-batch-requests" + prodSubscription = "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker" + prodBigQueryDataset = "scorecardcron" + prodBigQueryTable = "scorecard-v2" + prodCompletionThreshold = 0.99 + prodWebhookURL = "" + prodCIIDataBucket = "gs://ossf-scorecard-cii-data" + prodBlacklistedChecks = "CI-Tests,Contributors" + prodShardSize int = 10 + prodMetricExporter string = "stackdriver" + prodMetricStackdriverPrefix string = "scorecard-cron" // Raw results. prodRawBucket = "gs://ossf-scorecard-rawdata" prodRawBigQueryTable = "scorecard-rawdata" @@ -94,22 +95,22 @@ func TestYAMLParsing(t *testing.T) { name: "validate", filename: "config.yaml", expectedConfig: config{ - ProjectID: prodProjectID, - ResultDataBucketURL: prodBucket, - RequestTopicURL: prodTopic, - RequestSubscriptionURL: prodSubscription, - BigQueryDataset: prodBigQueryDataset, - BigQueryTable: prodBigQueryTable, - CompletionThreshold: prodCompletionThreshold, - WebhookURL: prodWebhookURL, - ShardSize: prodShardSize, - MetricExporter: prodMetricExporter, - InputBucketURL: prodInputBucketURL, - InputBucketPrefix: prodInputBucketPrefix, - AdditionalParams: prodAdditionalParams, + ProjectID: prodProjectID, + ResultDataBucketURL: prodBucket, + RequestTopicURL: prodTopic, + RequestSubscriptionURL: prodSubscription, + BigQueryDataset: prodBigQueryDataset, + BigQueryTable: prodBigQueryTable, + CompletionThreshold: prodCompletionThreshold, + WebhookURL: prodWebhookURL, + ShardSize: prodShardSize, + MetricExporter: prodMetricExporter, + MetricStackdriverPrefix: prodMetricStackdriverPrefix, + InputBucketURL: prodInputBucketURL, + InputBucketPrefix: prodInputBucketPrefix, + AdditionalParams: prodAdditionalParams, }, }, - { name: "basic", filename: "testdata/basic.yaml", @@ -393,6 +394,20 @@ func TestGetMetricExporter(t *testing.T) { }) } +//nolint:paralleltest // Since os.Setenv is used. +func TestGetMetricStackdriverPrefix(t *testing.T) { + t.Run("GetMetricStackdriverPrefix", func(t *testing.T) { + os.Unsetenv(metricStackdriverPrefix) + prefix, err := GetMetricStackdriverPrefix() + if err != nil { + t.Errorf("failed to get production metric stackdriver prefix from config: %v", err) + } + if prefix != prodMetricStackdriverPrefix { + t.Errorf("test failed: expected - %s, got = %s", prodMetricStackdriverPrefix, prefix) + } + }) +} + //nolint:paralleltest // Since os.Setenv is used. func TestGetAPIResultsBucketURL(t *testing.T) { t.Run("GetBigQueryExportsBucketURL", func(t *testing.T) { diff --git a/cron/internal/monitoring/exporter.go b/cron/internal/monitoring/exporter.go index 922ce1b4148e..84721d565256 100644 --- a/cron/internal/monitoring/exporter.go +++ b/cron/internal/monitoring/exporter.go @@ -32,7 +32,6 @@ var errorUndefinedExporter = errors.New("unsupported exporterType") type exporterType string const ( - stackdriverMetricPrefix = "scorecard-cron" stackdriverTimeSeriesQuota = 200 stackdriverTimeoutMinutes = 10 stackDriver exporterType = "stackdriver" @@ -69,9 +68,13 @@ func newStackDriverExporter() (*stackdriver.Exporter, error) { if err != nil { return nil, fmt.Errorf("error getting ProjectID: %w", err) } + prefix, err := config.GetMetricStackdriverPrefix() + if err != nil { + return nil, fmt.Errorf("error getting stackdriver prefix: %w", err) + } exporter, err := stackdriver.NewExporter(stackdriver.Options{ ProjectID: projectID, - MetricPrefix: stackdriverMetricPrefix, + MetricPrefix: prefix, MonitoredResource: gcp.Autodetect(), Timeout: stackdriverTimeoutMinutes * time.Minute, // Stackdriver specific quotas based on https://cloud.google.com/monitoring/quotas