From 03def1c29496d28ffc71e5537b6c5e6f8704e249 Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Thu, 17 Mar 2022 01:37:40 +0000 Subject: [PATCH 01/12] draft --- cron/config/config.go | 20 ++++++++++++- cron/config/config.yaml | 3 ++ cron/config/config_test.go | 9 ++++-- cron/controller/main.go | 19 +++++++++++- cron/k8s/controller.release.yaml | 10 +++++++ cron/k8s/raw-transfer-v0.yaml | 41 +++++++++++++++++++++++++ cron/k8s/transfer.raw-release-v0.yaml | 43 +++++++++++++++++++++++++++ cron/k8s/worker.release.yaml | 2 ++ cron/worker/main.go | 24 ++++++++++++--- 9 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 cron/k8s/raw-transfer-v0.yaml create mode 100644 cron/k8s/transfer.raw-release-v0.yaml diff --git a/cron/config/config.go b/cron/config/config.go index da18f96af40..bdf745c07c7 100644 --- a/cron/config/config.go +++ b/cron/config/config.go @@ -47,9 +47,12 @@ const ( metricExporter string = "SCORECARD_METRIC_EXPORTER" ciiDataBucketURL string = "SCORECARD_CII_DATA_BUCKET_URL" blacklistedChecks string = "SCORECARD_BLACKLISTED_CHECKS" - + // v2 results. bigqueryTableV2 string = "SCORECARD_BIGQUERY_TABLEV2" resultDataBucketURLV2 string = "SCORECARD_DATA_BUCKET_URLV2" + // v0 raw results. + rawBigqueryTableV0 string = "RAW_SCORECARD_BIGQUERY_TABLEV0" + rawResultDataBucketURLV0 string = "RAW_SCORECARD_DATA_BUCKET_URLV0" ) var ( @@ -78,6 +81,9 @@ type config struct { // UPGRADEv2: to remove. ResultDataBucketURLV2 string `yaml:"result-data-bucket-url-v2"` BigQueryTableV2 string `yaml:"bigquery-table-v2"` + // Raw results v0. + RawResultDataBucketURLV0 string `yaml:"raw-result-data-bucket-url-v0"` + RawBigQueryTableV0 string `yaml:"raw-bigquery-table-v0"` } func getParsedConfigFromFile(byteValue []byte) (config, error) { @@ -196,6 +202,18 @@ func GetResultDataBucketURLV2() (string, error) { return getStringConfigValue(resultDataBucketURLV2, configYAML, "ResultDataBucketURLV2", "result-data-bucket-url-v2") } +// GetRawBigQueryTableV0 returns the table name to transfer cron job results. +func GetRawBigQueryTableV0() (string, error) { + return getStringConfigValue(rawBigqueryTableV0, configYAML, + "RawBigQueryTableV0", "raw-bigquery-table-v0") +} + +// GetRawResultDataBucketURLV0 returns the bucketURL for storing cron job's raw results. +func GetRawResultDataBucketURLV0() (string, error) { + return getStringConfigValue(rawResultDataBucketURLV0, configYAML, + "RawResultDataBucketURLV0", "raw-result-data-bucket-url-v0") +} + // GetShardSize returns the shard_size for the cron job. func GetShardSize() (int, error) { return getIntConfigValue(shardSize, configYAML, "ShardSize", "shard-size") diff --git a/cron/config/config.yaml b/cron/config/config.yaml index 58c5874631e..3f57aad012e 100644 --- a/cron/config/config.yaml +++ b/cron/config/config.yaml @@ -29,3 +29,6 @@ metric-exporter: stackdriver # UPGRADEv2: to remove. result-data-bucket-url-v2: gs://ossf-scorecard-data2 bigquery-table-v2: scorecard-v2 +# Raw results. +raw-result-data-bucket-url-v0: gs://ossf-scorecard-rawdata-v0 +raw-bigquery-table-v0: scorecard-rawdata-v0 \ No newline at end of file diff --git a/cron/config/config_test.go b/cron/config/config_test.go index 6c83e2ae880..f5351ab6134 100644 --- a/cron/config/config_test.go +++ b/cron/config/config_test.go @@ -39,6 +39,9 @@ const ( // UPGRADEv2: to remove. prodBucketV2 = "gs://ossf-scorecard-data2" prodBigQueryTableV2 = "scorecard-v2" + // v0 raw results. + prodRawBucketV0 = "gs://ossf-scorecard-rawdata-v0" + prodRawBigQueryTableV0 = "scorecard-rawdata-v0" ) func getByteValueFromFile(filename string) ([]byte, error) { @@ -73,8 +76,10 @@ func TestYAMLParsing(t *testing.T) { ShardSize: prodShardSize, MetricExporter: prodMetricExporter, // UPGRADEv2: to remove. - ResultDataBucketURLV2: prodBucketV2, - BigQueryTableV2: prodBigQueryTableV2, + ResultDataBucketURLV2: prodBucketV2, + BigQueryTableV2: prodBigQueryTableV2, + RawResultDataBucketURLV0: prodRawBucketV0, + RawBigQueryTableV0: prodRawBigQueryTableV0, }, }, diff --git a/cron/controller/main.go b/cron/controller/main.go index dd43fa70889..bf9f6e172e5 100644 --- a/cron/controller/main.go +++ b/cron/controller/main.go @@ -34,7 +34,8 @@ import ( var headSHA = clients.HeadSHA func publishToRepoRequestTopic(iter data.Iterator, topicPublisher pubsub.Publisher, - shardSize int, datetime time.Time) (int32, error) { + shardSize int, datetime time.Time, +) (int32, error) { var shardNum int32 request := data.ScorecardBatchRequest{ JobTime: timestamppb.New(datetime), @@ -122,6 +123,11 @@ func main() { panic(err) } + rawBucketV0, err := config.GetRawResultDataBucketURLV0() + if err != nil { + panic(err) + } + shardNum, err := publishToRepoRequestTopic(reader, topicPublisher, shardSize, t) if err != nil { panic(err) @@ -154,4 +160,15 @@ func main() { if err != nil { panic(fmt.Errorf("error writing to BlobStore2: %w", err)) } + + // Raw data v0. + *metadata.ShardLoc = rawBucketV0 + "/" + data.GetBlobFilename("", t) + metadataJSON, err = protojson.Marshal(&metadata) + if err != nil { + panic(fmt.Errorf("error during protojson.Marshal raw-v0: %w", err)) + } + err = data.WriteToBlobStore(ctx, rawBucketV0, data.GetShardMetadataFilename(t), metadataJSON) + if err != nil { + panic(fmt.Errorf("error writing to BlobStore raw v0: %w", err)) + } } diff --git a/cron/k8s/controller.release.yaml b/cron/k8s/controller.release.yaml index f00cced6e80..d246f545bdf 100644 --- a/cron/k8s/controller.release.yaml +++ b/cron/k8s/controller.release.yaml @@ -69,6 +69,16 @@ spec: memory: 1Gi requests: memory: 1Gi + # Raw results v0. + - name: RAW_SCORECARD_DATA_BUCKET_URLV0 + value: "gs://ossf-scorecard-rawdata-releasetest-v0" + - name: SCORECARD_SHARD_SIZE + value: "5" + resources: + limits: + memory: 1Gi + requests: + memory: 1Gi - name: worker-update image: bitnami/kubectl@sha256:44468c0f5b348e6dcf5e11feb6fdcc969c874bba2856150fe50eb1aacb3bdfee command: diff --git a/cron/k8s/raw-transfer-v0.yaml b/cron/k8s/raw-transfer-v0.yaml new file mode 100644 index 00000000000..d2b57d851db --- /dev/null +++ b/cron/k8s/raw-transfer-v0.yaml @@ -0,0 +1,41 @@ +# Copyright 2021 Security Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: CronJob +metadata: + name: scorecard-bq-raw-transfer-v0 +spec: + # At 02:00UTC on Monday and Thursday. + schedule: "0 2 * * 1,4" + concurrencyPolicy: "Forbid" + jobTemplate: + spec: + template: + spec: + containers: + - name: bq-raw-transfer-v0 + image: gcr.io/openssf/scorecard-bq-transfer:latest + imagePullPolicy: Always + resources: + limits: + memory: 1Gi + requests: + memory: 1Gi + env: + - name: RAW_SCORECARD_BIGQUERY_TABLEV0 + value: "scorecard-rawdata-v0" + - name: RAW_SCORECARD_DATA_BUCKET_URLV0 + value: "gs://ossf-scorecard-rawdata-v0" + restartPolicy: OnFailure diff --git a/cron/k8s/transfer.raw-release-v0.yaml b/cron/k8s/transfer.raw-release-v0.yaml new file mode 100644 index 00000000000..1c674bf5f8b --- /dev/null +++ b/cron/k8s/transfer.raw-release-v0.yaml @@ -0,0 +1,43 @@ +# Copyright 2021 Security Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1 +kind: CronJob +metadata: + name: scorecard-bq-transfer-raw-releasetest-v0 +spec: + # Daily at 9am and 6pm PST. + schedule: "0 1,16 * * *" + concurrencyPolicy: "Forbid" + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: bq-transfer-raw-releasetest-v0 + image: gcr.io/openssf/scorecard-bq-transfer:latest + imagePullPolicy: Always + env: + - name: RAW_SCORECARD_DATA_BUCKET_URLV0 + value: "gs://ossf-scorecard-rawdata-releasetest-v0" + - name: RAW_SCORECARD_BIGQUERY_TABLEV0 + value: "scorecard_raw-releasetest-v0" + - name: SCORECARD_COMPLETION_THRESHOLD + value: "0.9" + resources: + limits: + memory: 1Gi + requests: + memory: 1Gi diff --git a/cron/k8s/worker.release.yaml b/cron/k8s/worker.release.yaml index 38c8492ac37..2a55d4223aa 100644 --- a/cron/k8s/worker.release.yaml +++ b/cron/k8s/worker.release.yaml @@ -37,6 +37,8 @@ spec: # UPGRADEv2: to remove. - name: SCORECARD_DATA_BUCKET_URLV2 value: "gs://ossf-scorecard-data-releasetest2" + - name: RAW_SCORECARD_DATA_BUCKET_URLV0 + value: "gs://ossf-scorecard-rawdata-releasetest-v0" - name: SCORECARD_REQUEST_SUBSCRIPTION_URL value: "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker-releasetest" - name: SCORECARD_BLACKLISTED_CHECKS diff --git a/cron/worker/main.go b/cron/worker/main.go index b0eafd425db..288ae266f7d 100644 --- a/cron/worker/main.go +++ b/cron/worker/main.go @@ -48,12 +48,13 @@ var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to tru // nolint: gocognit func processRequest(ctx context.Context, batchRequest *data.ScorecardBatchRequest, - blacklistedChecks []string, bucketURL, bucketURL2 string, + blacklistedChecks []string, bucketURL, bucketURL2, rawBucketURLV0 string, checkDocs docs.Doc, repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient, ciiClient clients.CIIBestPracticesClient, vulnsClient clients.VulnerabilitiesClient, - logger *log.Logger) error { + logger *log.Logger, +) error { filename := data.GetBlobFilename( fmt.Sprintf("shard-%07d", batchRequest.GetShardNum()), batchRequest.GetJobTime().AsTime()) @@ -67,7 +68,13 @@ func processRequest(ctx context.Context, if err != nil { return fmt.Errorf("error during BlobExists: %w", err) } - if exists1 && exists2 { + + exists3, err := data.BlobExists(ctx, rawBucketURLV0, filename) + if err != nil { + return fmt.Errorf("error during BlobExists: %w", err) + } + + if exists1 && exists2 && exists3 { logger.Info(fmt.Sprintf("Already processed shard %s. Nothing to do.", filename)) // We have already processed this request, nothing to do. return nil @@ -129,6 +136,8 @@ func processRequest(ctx context.Context, if err := format.AsJSON2(&result, true /*showDetails*/, log.InfoLevel, checkDocs, &buffer2); err != nil { return fmt.Errorf("error during result.AsJSON2: %w", err) } + + // TODO: raw result v0. } if err := data.WriteToBlobStore(ctx, bucketURL, filename, buffer.Bytes()); err != nil { return fmt.Errorf("error during WriteToBlobStore: %w", err) @@ -138,6 +147,8 @@ func processRequest(ctx context.Context, return fmt.Errorf("error during WriteToBlobStore2: %w", err) } + // TODO: raw result v0. + logger.Info(fmt.Sprintf("Write to shard file successful: %s", filename)) return nil @@ -191,6 +202,11 @@ func main() { panic(err) } + rawBucketURLV0, err := config.GetRawResultDataBucketURLV0() + if err != nil { + panic(err) + } + blacklistedChecks, err := config.GetBlacklistedChecks() if err != nil { panic(err) @@ -236,7 +252,7 @@ func main() { break } if err := processRequest(ctx, req, blacklistedChecks, - bucketURL, bucketURL2, checkDocs, + bucketURL, bucketURL2, rawBucketURLV0, checkDocs, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil { // TODO(log): Previously Warn. Consider logging an error here. logger.Info(fmt.Sprintf("error processing request: %v", err)) From 627e1fb307ff9e6c11f355c81e9ffb8c9c7f9172 Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Thu, 17 Mar 2022 02:12:45 +0000 Subject: [PATCH 02/12] updates --- checks/binary_artifact.go | 3 +-- checks/branch_protection.go | 1 - checks/code_review.go | 1 - checks/dependency_update_tool.go | 3 +-- checks/security_policy.go | 1 - checks/vulnerabilities.go | 1 - cmd/root.go | 1 - cmd/serve.go | 2 +- cron/worker/main.go | 2 +- pkg/scorecard.go | 16 ++++++---------- 10 files changed, 10 insertions(+), 21 deletions(-) diff --git a/checks/binary_artifact.go b/checks/binary_artifact.go index a3d0d567871..bb0312d458d 100644 --- a/checks/binary_artifact.go +++ b/checks/binary_artifact.go @@ -26,7 +26,7 @@ const CheckBinaryArtifacts string = "Binary-Artifacts" //nolint func init() { - var supportedRequestTypes = []checker.RequestType{ + supportedRequestTypes := []checker.RequestType{ checker.FileBased, checker.CommitBased, } @@ -47,7 +47,6 @@ func BinaryArtifacts(c *checker.CheckRequest) checker.CheckResult { // Return raw results. if c.RawResults != nil { c.RawResults.BinaryArtifactResults = rawData - return checker.CheckResult{} } // Return the score evaluation. diff --git a/checks/branch_protection.go b/checks/branch_protection.go index 2ab0a05769c..a45bac720b3 100644 --- a/checks/branch_protection.go +++ b/checks/branch_protection.go @@ -43,7 +43,6 @@ func BranchProtection(c *checker.CheckRequest) checker.CheckResult { // Return raw results. if c.RawResults != nil { c.RawResults.BranchProtectionResults = rawData - return checker.CheckResult{} } // Return the score evaluation. diff --git a/checks/code_review.go b/checks/code_review.go index 91c6ad9aba2..11bfff72918 100644 --- a/checks/code_review.go +++ b/checks/code_review.go @@ -46,7 +46,6 @@ func CodeReview(c *checker.CheckRequest) checker.CheckResult { // Return raw results. if c.RawResults != nil { c.RawResults.CodeReviewResults = rawData - return checker.CheckResult{} } // Return the score evaluation. diff --git a/checks/dependency_update_tool.go b/checks/dependency_update_tool.go index 286dd6d5c48..5f0f2b490a9 100644 --- a/checks/dependency_update_tool.go +++ b/checks/dependency_update_tool.go @@ -26,7 +26,7 @@ const CheckDependencyUpdateTool = "Dependency-Update-Tool" //nolint func init() { - var supportedRequestTypes = []checker.RequestType{ + supportedRequestTypes := []checker.RequestType{ checker.FileBased, } if err := registerCheck(CheckDependencyUpdateTool, DependencyUpdateTool, supportedRequestTypes); err != nil { @@ -46,7 +46,6 @@ func DependencyUpdateTool(c *checker.CheckRequest) checker.CheckResult { // Return raw results. if c.RawResults != nil { c.RawResults.DependencyUpdateToolResults = rawData - return checker.CheckResult{} } // Return the score evaluation. diff --git a/checks/security_policy.go b/checks/security_policy.go index 0505a58cf6e..5d8ec594a14 100644 --- a/checks/security_policy.go +++ b/checks/security_policy.go @@ -47,7 +47,6 @@ func SecurityPolicy(c *checker.CheckRequest) checker.CheckResult { // Set the raw results. if c.RawResults != nil { c.RawResults.SecurityPolicyResults = rawData - return checker.CheckResult{} } return evaluation.SecurityPolicy(CheckSecurityPolicy, c.Dlogger, &rawData) diff --git a/checks/vulnerabilities.go b/checks/vulnerabilities.go index 94706c49680..22944b48bca 100644 --- a/checks/vulnerabilities.go +++ b/checks/vulnerabilities.go @@ -46,7 +46,6 @@ func Vulnerabilities(c *checker.CheckRequest) checker.CheckResult { // Set the raw results. if c.RawResults != nil { c.RawResults.VulnerabilitiesResults = rawData - return checker.CheckResult{} } return evaluation.Vulnerabilities(CheckVulnerabilities, c.Dlogger, &rawData) diff --git a/cmd/root.go b/cmd/root.go index e8f54561678..cf8c1e59406 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -128,7 +128,6 @@ func rootCmd(o *options.Options) { ctx, repoURI, o.Commit, - o.Format == options.FormatRaw, enabledChecks, repoClient, ossFuzzRepoClient, diff --git a/cmd/serve.go b/cmd/serve.go index a57765fd79c..8d4f2f4a3a1 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -69,7 +69,7 @@ func serveCmd(o *options.Options) *cobra.Command { defer ossFuzzRepoClient.Close() ciiClient := clients.DefaultCIIBestPracticesClient() repoResult, err := pkg.RunScorecards( - ctx, repo, clients.HeadSHA /*commitSHA*/, false /*raw*/, checks.AllChecks, repoClient, + ctx, repo, clients.HeadSHA /*commitSHA*/, checks.AllChecks, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient) if err != nil { logger.Error(err, "running enabled scorecard checks on repo") diff --git a/cron/worker/main.go b/cron/worker/main.go index 288ae266f7d..948f057a003 100644 --- a/cron/worker/main.go +++ b/cron/worker/main.go @@ -106,7 +106,7 @@ func processRequest(ctx context.Context, for _, check := range blacklistedChecks { delete(checksToRun, check) } - result, err := pkg.RunScorecards(ctx, repo, commitSHA, false /*raw*/, checksToRun, + result, err := pkg.RunScorecards(ctx, repo, commitSHA, checksToRun, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient) if errors.Is(err, sce.ErrRepoUnreachable) { // Not accessible repo - continue. diff --git a/pkg/scorecard.go b/pkg/scorecard.go index cfda7297637..0cc42b5f87f 100644 --- a/pkg/scorecard.go +++ b/pkg/scorecard.go @@ -31,7 +31,8 @@ func runEnabledChecks(ctx context.Context, repo clients.Repo, raw *checker.RawResults, checksToRun checker.CheckNameToFnMap, repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient, ciiClient clients.CIIBestPracticesClient, vulnsClient clients.VulnerabilitiesClient, - resultsCh chan checker.CheckResult) { + resultsCh chan checker.CheckResult, +) { request := checker.CheckRequest{ Ctx: ctx, RepoClient: repoClient, @@ -78,12 +79,12 @@ func getRepoCommitHash(r clients.RepoClient) (string, error) { func RunScorecards(ctx context.Context, repo clients.Repo, commitSHA string, - raw bool, checksToRun checker.CheckNameToFnMap, repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient, ciiClient clients.CIIBestPracticesClient, - vulnsClient clients.VulnerabilitiesClient) (ScorecardResult, error) { + vulnsClient clients.VulnerabilitiesClient, +) (ScorecardResult, error) { if err := repoClient.InitRepo(repo, commitSHA); err != nil { // No need to call sce.WithMessage() since InitRepo will do that for us. //nolint:wrapcheck @@ -108,13 +109,8 @@ func RunScorecards(ctx context.Context, Date: time.Now(), } resultsCh := make(chan checker.CheckResult) - if raw { - go runEnabledChecks(ctx, repo, &ret.RawResults, checksToRun, repoClient, ossFuzzRepoClient, - ciiClient, vulnsClient, resultsCh) - } else { - go runEnabledChecks(ctx, repo, nil, checksToRun, repoClient, ossFuzzRepoClient, - ciiClient, vulnsClient, resultsCh) - } + go runEnabledChecks(ctx, repo, &ret.RawResults, checksToRun, repoClient, ossFuzzRepoClient, + ciiClient, vulnsClient, resultsCh) for result := range resultsCh { ret.Checks = append(ret.Checks, result) From 72511d87f9e82d1b675a8ba57accae452f4ece04 Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Thu, 17 Mar 2022 03:51:01 +0000 Subject: [PATCH 03/12] updates --- cron/config/config.yaml | 4 ++-- cron/worker/main.go | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cron/config/config.yaml b/cron/config/config.yaml index 3f57aad012e..04a1520dac1 100644 --- a/cron/config/config.yaml +++ b/cron/config/config.yaml @@ -28,7 +28,7 @@ blacklisted-checks: SAST,CI-Tests,Contributors metric-exporter: stackdriver # UPGRADEv2: to remove. result-data-bucket-url-v2: gs://ossf-scorecard-data2 -bigquery-table-v2: scorecard-v2 +bigquery-table-v2: scorecard-v2ƒ # Raw results. raw-result-data-bucket-url-v0: gs://ossf-scorecard-rawdata-v0 -raw-bigquery-table-v0: scorecard-rawdata-v0 \ No newline at end of file +raw-bigquery-table-v0: scorecard-rawdata-v0 diff --git a/cron/worker/main.go b/cron/worker/main.go index 948f057a003..9f47be836bd 100644 --- a/cron/worker/main.go +++ b/cron/worker/main.go @@ -82,6 +82,7 @@ func processRequest(ctx context.Context, var buffer bytes.Buffer var buffer2 bytes.Buffer + var rawBufferV0 bytes.Buffer // TODO: run Scorecard for each repo in a separate thread. for _, repoReq := range batchRequest.GetRepos() { logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", *repoReq.Url)) @@ -137,7 +138,11 @@ func processRequest(ctx context.Context, return fmt.Errorf("error during result.AsJSON2: %w", err) } - // TODO: raw result v0. + // Raw result v0. + if err := format.AsRawJSON(&result, &rawBufferV0); err != nil { + return fmt.Errorf("error during result.AsRawJSON: %w", err) + } + } if err := data.WriteToBlobStore(ctx, bucketURL, filename, buffer.Bytes()); err != nil { return fmt.Errorf("error during WriteToBlobStore: %w", err) @@ -147,7 +152,10 @@ func processRequest(ctx context.Context, return fmt.Errorf("error during WriteToBlobStore2: %w", err) } - // TODO: raw result v0. + // Raw result v0. + if err := data.WriteToBlobStore(ctx, rawBucketURLV0, filename, rawBufferV0.Bytes()); err != nil { + return fmt.Errorf("error during WriteToBlobStore2: %w", err) + } logger.Info(fmt.Sprintf("Write to shard file successful: %s", filename)) From e2ac020cd8deedde7480ec9b227cb84ec73839a4 Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Thu, 17 Mar 2022 03:53:45 +0000 Subject: [PATCH 04/12] updates --- cron/format/json_raw_results.go | 308 ++++++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 cron/format/json_raw_results.go diff --git a/cron/format/json_raw_results.go b/cron/format/json_raw_results.go new file mode 100644 index 00000000000..93189a7b373 --- /dev/null +++ b/cron/format/json_raw_results.go @@ -0,0 +1,308 @@ +// Copyright 2021 Security Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package format + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/ossf/scorecard/v4/checker" + sce "github.com/ossf/scorecard/v4/errors" + "github.com/ossf/scorecard/v4/pkg" +) + +// Flat JSON structure to hold raw results. +type jsonScorecardRawResult struct { + Date string `json:"date"` + Repo jsonRepoV2 `json:"repo"` + Scorecard jsonScorecardV2 `json:"scorecard"` + Metadata []string `json:"metadata"` + Results jsonRawResults `json:"results"` +} + +// TODO: separate each check extraction into its own file. +type jsonFile struct { + Path string `json:"path"` + Offset int `json:"offset,omitempty"` +} + +type jsonTool struct { + Name string `json:"name"` + URL string `json:"url"` + Desc string `json:"desc"` + ConfigFiles []jsonFile `json:"files"` + // TODO: Runs, Issues, Merge requests. +} + +type jsonBranchProtectionSettings struct { + RequiredApprovingReviewCount *int `json:"required-reviewer-count"` + AllowsDeletions *bool `json:"allows-deletions"` + AllowsForcePushes *bool `json:"allows-force-pushes"` + RequiresCodeOwnerReviews *bool `json:"requires-code-owner-review"` + RequiresLinearHistory *bool `json:"required-linear-history"` + DismissesStaleReviews *bool `json:"dismisses-stale-reviews"` + EnforcesAdmins *bool `json:"enforces-admin"` + RequiresStatusChecks *bool `json:"requires-status-checks"` + RequiresUpToDateBranchBeforeMerging *bool `json:"requires-updated-branches-to-merge"` + StatusCheckContexts []string `json:"status-checks-contexts"` +} + +type jsonBranchProtection struct { + Protection *jsonBranchProtectionSettings `json:"protection"` + Name string `json:"name"` +} + +type jsonReview struct { + Reviewer jsonUser `json:"reviewer"` + State string `json:"state"` +} + +type jsonUser struct { + Login string `json:"login"` +} + +//nolint:govet +type jsonMergeRequest struct { + Number int `json:"number"` + Labels []string `json:"labels"` + Reviews []jsonReview `json:"reviews"` + Author jsonUser `json:"author"` +} + +type jsonDefaultBranchCommit struct { + // ApprovedReviews *jsonApprovedReviews `json:"approved-reviews"` + Committer jsonUser `json:"committer"` + MergeRequest *jsonMergeRequest `json:"merge-request"` + CommitMessage string `json:"commit-message"` + SHA string `json:"sha"` + + // TODO: check runs, etc. +} + +type jsonRawResults struct { + DatabaseVulnerabilities []jsonDatabaseVulnerability `json:"database-vulnerabilities"` + // List of binaries found in the repo. + Binaries []jsonFile `json:"binaries"` + // List of security policy files found in the repo. + // Note: we return one at most. + SecurityPolicies []jsonFile `json:"security-policies"` + // List of update tools. + // Note: we return one at most. + DependencyUpdateTools []jsonTool `json:"dependency-update-tools"` + // Branch protection settings for development and release branches. + BranchProtections []jsonBranchProtection `json:"branch-protections"` + // Commits. + DefaultBranchCommits []jsonDefaultBranchCommit `json:"default-branch-commits"` +} + +//nolint:unparam +func addCodeReviewRawResults(r *jsonScorecardRawResult, cr *checker.CodeReviewData) error { + r.Results.DefaultBranchCommits = []jsonDefaultBranchCommit{} + for _, commit := range cr.DefaultBranchCommits { + com := jsonDefaultBranchCommit{ + Committer: jsonUser{ + Login: commit.Committer.Login, + }, + CommitMessage: commit.CommitMessage, + SHA: commit.SHA, + } + + // Merge request field. + if commit.MergeRequest != nil { + mr := jsonMergeRequest{ + Number: commit.MergeRequest.Number, + Author: jsonUser{ + Login: commit.MergeRequest.Author.Login, + }, + } + + if len(commit.MergeRequest.Labels) > 0 { + mr.Labels = commit.MergeRequest.Labels + } + + for _, r := range commit.MergeRequest.Reviews { + mr.Reviews = append(mr.Reviews, jsonReview{ + State: r.State, + Reviewer: jsonUser{ + Login: r.Reviewer.Login, + }, + }) + } + + com.MergeRequest = &mr + } + + com.CommitMessage = commit.CommitMessage + + r.Results.DefaultBranchCommits = append(r.Results.DefaultBranchCommits, com) + } + return nil +} + +type jsonDatabaseVulnerability struct { + // For OSV: OSV-2020-484 + // For CVE: CVE-2022-23945 + ID string + // TODO: additional information +} + +//nolint:unparam +func addVulnerbilitiesRawResults(r *jsonScorecardRawResult, vd *checker.VulnerabilitiesData) error { + r.Results.DatabaseVulnerabilities = []jsonDatabaseVulnerability{} + for _, v := range vd.Vulnerabilities { + r.Results.DatabaseVulnerabilities = append(r.Results.DatabaseVulnerabilities, + jsonDatabaseVulnerability{ + ID: v.ID, + }) + } + return nil +} + +//nolint:unparam +func addBinaryArtifactRawResults(r *jsonScorecardRawResult, ba *checker.BinaryArtifactData) error { + r.Results.Binaries = []jsonFile{} + for _, v := range ba.Files { + r.Results.Binaries = append(r.Results.Binaries, jsonFile{ + Path: v.Path, + }) + } + return nil +} + +//nolint:unparam +func addSecurityPolicyRawResults(r *jsonScorecardRawResult, sp *checker.SecurityPolicyData) error { + r.Results.SecurityPolicies = []jsonFile{} + for _, v := range sp.Files { + r.Results.SecurityPolicies = append(r.Results.SecurityPolicies, jsonFile{ + Path: v.Path, + }) + } + return nil +} + +//nolint:unparam +func addDependencyUpdateToolRawResults(r *jsonScorecardRawResult, + dut *checker.DependencyUpdateToolData, +) error { + r.Results.DependencyUpdateTools = []jsonTool{} + for i := range dut.Tools { + t := dut.Tools[i] + offset := len(r.Results.DependencyUpdateTools) + r.Results.DependencyUpdateTools = append(r.Results.DependencyUpdateTools, jsonTool{ + Name: t.Name, + URL: t.URL, + Desc: t.Desc, + }) + for _, f := range t.ConfigFiles { + r.Results.DependencyUpdateTools[offset].ConfigFiles = append( + r.Results.DependencyUpdateTools[offset].ConfigFiles, + jsonFile{ + Path: f.Path, + }, + ) + } + } + return nil +} + +//nolint:unparam +func addBranchProtectionRawResults(r *jsonScorecardRawResult, bp *checker.BranchProtectionsData) error { + r.Results.BranchProtections = []jsonBranchProtection{} + for _, v := range bp.Branches { + var bp *jsonBranchProtectionSettings + if v.Protected != nil && *v.Protected { + bp = &jsonBranchProtectionSettings{ + AllowsDeletions: v.AllowsDeletions, + AllowsForcePushes: v.AllowsForcePushes, + RequiresCodeOwnerReviews: v.RequiresCodeOwnerReviews, + RequiresLinearHistory: v.RequiresLinearHistory, + DismissesStaleReviews: v.DismissesStaleReviews, + EnforcesAdmins: v.EnforcesAdmins, + RequiresStatusChecks: v.RequiresStatusChecks, + RequiresUpToDateBranchBeforeMerging: v.RequiresUpToDateBranchBeforeMerging, + RequiredApprovingReviewCount: v.RequiredApprovingReviewCount, + StatusCheckContexts: v.StatusCheckContexts, + } + } + r.Results.BranchProtections = append(r.Results.BranchProtections, jsonBranchProtection{ + Name: v.Name, + Protection: bp, + }) + } + return nil +} + +func fillJSONRawResults(r *jsonScorecardRawResult, raw *checker.RawResults) error { + // Vulnerabiliries. + if err := addVulnerbilitiesRawResults(r, &raw.VulnerabilitiesResults); err != nil { + return sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + } + + // Binary-Artifacts. + if err := addBinaryArtifactRawResults(r, &raw.BinaryArtifactResults); err != nil { + return sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + } + + // Security-Policy. + if err := addSecurityPolicyRawResults(r, &raw.SecurityPolicyResults); err != nil { + return sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + } + + // Dependency-Update-Tool. + if err := addDependencyUpdateToolRawResults(r, &raw.DependencyUpdateToolResults); err != nil { + return sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + } + + // Branch-Protection. + if err := addBranchProtectionRawResults(r, &raw.BranchProtectionResults); err != nil { + return sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + } + + // Code-Review. + if err := addCodeReviewRawResults(r, &raw.CodeReviewResults); err != nil { + return sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + } + + return nil +} + +// AsRawJSON exports results as JSON for raw results. +func AsRawJSON(r *pkg.ScorecardResult, writer io.Writer) error { + encoder := json.NewEncoder(writer) + out := jsonScorecardRawResult{ + Repo: jsonRepoV2{ + Name: r.Repo.Name, + Commit: r.Repo.CommitSHA, + }, + Scorecard: jsonScorecardV2{ + Version: r.Scorecard.Version, + Commit: r.Scorecard.CommitSHA, + }, + Date: r.Date.Format("2006-01-02"), + Metadata: r.Metadata, + } + + // if err := out.fillJSONRawResults(r.Checks[0].RawResults); err != nil { + if err := fillJSONRawResults(&out, &r.RawResults); err != nil { + return err + } + + if err := encoder.Encode(out); err != nil { + return sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("encoder.Encode: %v", err)) + } + + return nil +} From 8c7516a6ea8b5893688c77bf3ccaf6fc8f0bfa3a Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Thu, 17 Mar 2022 03:54:39 +0000 Subject: [PATCH 05/12] updates --- cron/format/json_raw_results.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/format/json_raw_results.go b/cron/format/json_raw_results.go index 93189a7b373..ed0d42cb9a3 100644 --- a/cron/format/json_raw_results.go +++ b/cron/format/json_raw_results.go @@ -1,4 +1,4 @@ -// Copyright 2021 Security Scorecard Authors +// Copyright 2022 Security Scorecard Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From c581b3560c8532542ca469844b32499b38346247 Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Thu, 17 Mar 2022 04:06:42 +0000 Subject: [PATCH 06/12] updates --- cron/config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/config/config.yaml b/cron/config/config.yaml index 04a1520dac1..62ce6ab99ca 100644 --- a/cron/config/config.yaml +++ b/cron/config/config.yaml @@ -28,7 +28,7 @@ blacklisted-checks: SAST,CI-Tests,Contributors metric-exporter: stackdriver # UPGRADEv2: to remove. result-data-bucket-url-v2: gs://ossf-scorecard-data2 -bigquery-table-v2: scorecard-v2ƒ +bigquery-table-v2: scorecard-v2 # Raw results. raw-result-data-bucket-url-v0: gs://ossf-scorecard-rawdata-v0 raw-bigquery-table-v0: scorecard-rawdata-v0 From 36f28fcbf78ff53f294ec3ba096840ad98bc2693 Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Fri, 18 Mar 2022 19:40:08 +0000 Subject: [PATCH 07/12] comments --- cron/config/config.go | 24 +++++++++---------- cron/config/config_test.go | 8 +++---- cron/controller/main.go | 8 +++---- cron/k8s/controller.release.yaml | 9 +------ ...raw-transfer-v0.yaml => transfer-raw.yaml} | 4 ++-- ...ease-v0.yaml => transfer.release-raw.yaml} | 4 ++-- cron/k8s/worker.release.yaml | 2 +- cron/worker/main.go | 15 ++++++------ 8 files changed, 33 insertions(+), 41 deletions(-) rename cron/k8s/{raw-transfer-v0.yaml => transfer-raw.yaml} (92%) rename cron/k8s/{transfer.raw-release-v0.yaml => transfer.release-raw.yaml} (93%) diff --git a/cron/config/config.go b/cron/config/config.go index bdf745c07c7..d575117790a 100644 --- a/cron/config/config.go +++ b/cron/config/config.go @@ -51,8 +51,8 @@ const ( bigqueryTableV2 string = "SCORECARD_BIGQUERY_TABLEV2" resultDataBucketURLV2 string = "SCORECARD_DATA_BUCKET_URLV2" // v0 raw results. - rawBigqueryTableV0 string = "RAW_SCORECARD_BIGQUERY_TABLEV0" - rawResultDataBucketURLV0 string = "RAW_SCORECARD_DATA_BUCKET_URLV0" + rawBigqueryTable string = "RAW_SCORECARD_BIGQUERY_TABLE" + rawResultDataBucketURL string = "RAW_SCORECARD_DATA_BUCKET_URL" ) var ( @@ -82,8 +82,8 @@ type config struct { ResultDataBucketURLV2 string `yaml:"result-data-bucket-url-v2"` BigQueryTableV2 string `yaml:"bigquery-table-v2"` // Raw results v0. - RawResultDataBucketURLV0 string `yaml:"raw-result-data-bucket-url-v0"` - RawBigQueryTableV0 string `yaml:"raw-bigquery-table-v0"` + RawResultDataBucketURL string `yaml:"raw-result-data-bucket-url"` + RawBigQueryTable string `yaml:"raw-bigquery-table"` } func getParsedConfigFromFile(byteValue []byte) (config, error) { @@ -202,16 +202,16 @@ func GetResultDataBucketURLV2() (string, error) { return getStringConfigValue(resultDataBucketURLV2, configYAML, "ResultDataBucketURLV2", "result-data-bucket-url-v2") } -// GetRawBigQueryTableV0 returns the table name to transfer cron job results. -func GetRawBigQueryTableV0() (string, error) { - return getStringConfigValue(rawBigqueryTableV0, configYAML, - "RawBigQueryTableV0", "raw-bigquery-table-v0") +// GetRawBigQueryTable returns the table name to transfer cron job results. +func GetRawBigQueryTable() (string, error) { + return getStringConfigValue(rawBigqueryTable, configYAML, + "RawBigQueryTable", "raw-bigquery-table") } -// GetRawResultDataBucketURLV0 returns the bucketURL for storing cron job's raw results. -func GetRawResultDataBucketURLV0() (string, error) { - return getStringConfigValue(rawResultDataBucketURLV0, configYAML, - "RawResultDataBucketURLV0", "raw-result-data-bucket-url-v0") +// GetRawResultDataBucketURL returns the bucketURL for storing cron job's raw results. +func GetRawResultDataBucketURL() (string, error) { + return getStringConfigValue(rawResultDataBucketURL, configYAML, + "RawResultDataBucketURL", "raw-result-data-bucket-url") } // GetShardSize returns the shard_size for the cron job. diff --git a/cron/config/config_test.go b/cron/config/config_test.go index f5351ab6134..be4a43ebaa9 100644 --- a/cron/config/config_test.go +++ b/cron/config/config_test.go @@ -40,8 +40,8 @@ const ( prodBucketV2 = "gs://ossf-scorecard-data2" prodBigQueryTableV2 = "scorecard-v2" // v0 raw results. - prodRawBucketV0 = "gs://ossf-scorecard-rawdata-v0" - prodRawBigQueryTableV0 = "scorecard-rawdata-v0" + prodRawBucket = "gs://ossf-scorecard-rawdata" + prodRawBigQueryTable = "scorecard-rawdata" ) func getByteValueFromFile(filename string) ([]byte, error) { @@ -78,8 +78,8 @@ func TestYAMLParsing(t *testing.T) { // UPGRADEv2: to remove. ResultDataBucketURLV2: prodBucketV2, BigQueryTableV2: prodBigQueryTableV2, - RawResultDataBucketURLV0: prodRawBucketV0, - RawBigQueryTableV0: prodRawBigQueryTableV0, + RawResultDataBucketURL: prodRawBucket, + RawBigQueryTable: prodRawBigQueryTable, }, }, diff --git a/cron/controller/main.go b/cron/controller/main.go index bf9f6e172e5..ebd08f23ec3 100644 --- a/cron/controller/main.go +++ b/cron/controller/main.go @@ -123,7 +123,7 @@ func main() { panic(err) } - rawBucketV0, err := config.GetRawResultDataBucketURLV0() + rawBucket, err := config.GetRawResultDataBucketURL() if err != nil { panic(err) } @@ -162,12 +162,12 @@ func main() { } // Raw data v0. - *metadata.ShardLoc = rawBucketV0 + "/" + data.GetBlobFilename("", t) + *metadata.ShardLoc = rawBucket + "/" + data.GetBlobFilename("", t) metadataJSON, err = protojson.Marshal(&metadata) if err != nil { - panic(fmt.Errorf("error during protojson.Marshal raw-v0: %w", err)) + panic(fmt.Errorf("error during protojson.Marshal raw: %w", err)) } - err = data.WriteToBlobStore(ctx, rawBucketV0, data.GetShardMetadataFilename(t), metadataJSON) + err = data.WriteToBlobStore(ctx, rawBucket, data.GetShardMetadataFilename(t), metadataJSON) if err != nil { panic(fmt.Errorf("error writing to BlobStore raw v0: %w", err)) } diff --git a/cron/k8s/controller.release.yaml b/cron/k8s/controller.release.yaml index d246f545bdf..c4a5777f78b 100644 --- a/cron/k8s/controller.release.yaml +++ b/cron/k8s/controller.release.yaml @@ -62,15 +62,8 @@ spec: # UPGRADEv2: to remove. - name: SCORECARD_DATA_BUCKET_URLV2 value: "gs://ossf-scorecard-data-releasetest2" - - name: SCORECARD_SHARD_SIZE - value: "5" - resources: - limits: - memory: 1Gi - requests: - memory: 1Gi # Raw results v0. - - name: RAW_SCORECARD_DATA_BUCKET_URLV0 + - name: RAW_SCORECARD_DATA_BUCKET_URL value: "gs://ossf-scorecard-rawdata-releasetest-v0" - name: SCORECARD_SHARD_SIZE value: "5" diff --git a/cron/k8s/raw-transfer-v0.yaml b/cron/k8s/transfer-raw.yaml similarity index 92% rename from cron/k8s/raw-transfer-v0.yaml rename to cron/k8s/transfer-raw.yaml index d2b57d851db..7a38f741946 100644 --- a/cron/k8s/raw-transfer-v0.yaml +++ b/cron/k8s/transfer-raw.yaml @@ -34,8 +34,8 @@ spec: requests: memory: 1Gi env: - - name: RAW_SCORECARD_BIGQUERY_TABLEV0 + - name: RAW_SCORECARD_BIGQUERY_TABLE value: "scorecard-rawdata-v0" - - name: RAW_SCORECARD_DATA_BUCKET_URLV0 + - name: RAW_SCORECARD_DATA_BUCKET_URL value: "gs://ossf-scorecard-rawdata-v0" restartPolicy: OnFailure diff --git a/cron/k8s/transfer.raw-release-v0.yaml b/cron/k8s/transfer.release-raw.yaml similarity index 93% rename from cron/k8s/transfer.raw-release-v0.yaml rename to cron/k8s/transfer.release-raw.yaml index 1c674bf5f8b..6835a482c9a 100644 --- a/cron/k8s/transfer.raw-release-v0.yaml +++ b/cron/k8s/transfer.release-raw.yaml @@ -30,9 +30,9 @@ spec: image: gcr.io/openssf/scorecard-bq-transfer:latest imagePullPolicy: Always env: - - name: RAW_SCORECARD_DATA_BUCKET_URLV0 + - name: RAW_SCORECARD_DATA_BUCKET_URL value: "gs://ossf-scorecard-rawdata-releasetest-v0" - - name: RAW_SCORECARD_BIGQUERY_TABLEV0 + - name: RAW_SCORECARD_BIGQUERY_TABLE value: "scorecard_raw-releasetest-v0" - name: SCORECARD_COMPLETION_THRESHOLD value: "0.9" diff --git a/cron/k8s/worker.release.yaml b/cron/k8s/worker.release.yaml index 2a55d4223aa..78c4ec80bcf 100644 --- a/cron/k8s/worker.release.yaml +++ b/cron/k8s/worker.release.yaml @@ -37,7 +37,7 @@ spec: # UPGRADEv2: to remove. - name: SCORECARD_DATA_BUCKET_URLV2 value: "gs://ossf-scorecard-data-releasetest2" - - name: RAW_SCORECARD_DATA_BUCKET_URLV0 + - name: RAW_SCORECARD_DATA_BUCKET_URL value: "gs://ossf-scorecard-rawdata-releasetest-v0" - name: SCORECARD_REQUEST_SUBSCRIPTION_URL value: "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker-releasetest" diff --git a/cron/worker/main.go b/cron/worker/main.go index 9f47be836bd..c66cd3a4e5b 100644 --- a/cron/worker/main.go +++ b/cron/worker/main.go @@ -48,7 +48,7 @@ var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to tru // nolint: gocognit func processRequest(ctx context.Context, batchRequest *data.ScorecardBatchRequest, - blacklistedChecks []string, bucketURL, bucketURL2, rawBucketURLV0 string, + blacklistedChecks []string, bucketURL, bucketURL2, rawBucketURL string, checkDocs docs.Doc, repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient, ciiClient clients.CIIBestPracticesClient, @@ -69,7 +69,7 @@ func processRequest(ctx context.Context, return fmt.Errorf("error during BlobExists: %w", err) } - exists3, err := data.BlobExists(ctx, rawBucketURLV0, filename) + exists3, err := data.BlobExists(ctx, rawBucketURL, filename) if err != nil { return fmt.Errorf("error during BlobExists: %w", err) } @@ -82,7 +82,7 @@ func processRequest(ctx context.Context, var buffer bytes.Buffer var buffer2 bytes.Buffer - var rawBufferV0 bytes.Buffer + var rawBuffer bytes.Buffer // TODO: run Scorecard for each repo in a separate thread. for _, repoReq := range batchRequest.GetRepos() { logger.Info(fmt.Sprintf("Running Scorecard for repo: %s", *repoReq.Url)) @@ -139,10 +139,9 @@ func processRequest(ctx context.Context, } // Raw result v0. - if err := format.AsRawJSON(&result, &rawBufferV0); err != nil { + if err := format.AsRawJSON(&result, &rawBuffer); err != nil { return fmt.Errorf("error during result.AsRawJSON: %w", err) } - } if err := data.WriteToBlobStore(ctx, bucketURL, filename, buffer.Bytes()); err != nil { return fmt.Errorf("error during WriteToBlobStore: %w", err) @@ -153,7 +152,7 @@ func processRequest(ctx context.Context, } // Raw result v0. - if err := data.WriteToBlobStore(ctx, rawBucketURLV0, filename, rawBufferV0.Bytes()); err != nil { + if err := data.WriteToBlobStore(ctx, rawBucketURL, filename, rawBuffer.Bytes()); err != nil { return fmt.Errorf("error during WriteToBlobStore2: %w", err) } @@ -210,7 +209,7 @@ func main() { panic(err) } - rawBucketURLV0, err := config.GetRawResultDataBucketURLV0() + rawBucketURL, err := config.GetRawResultDataBucketURL() if err != nil { panic(err) } @@ -260,7 +259,7 @@ func main() { break } if err := processRequest(ctx, req, blacklistedChecks, - bucketURL, bucketURL2, rawBucketURLV0, checkDocs, + bucketURL, bucketURL2, rawBucketURL, checkDocs, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil { // TODO(log): Previously Warn. Consider logging an error here. logger.Info(fmt.Sprintf("error processing request: %v", err)) From 0fa9c3dc6f8441f9bf4ea3d7f07f8c4acb0ef23f Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Fri, 18 Mar 2022 19:42:26 +0000 Subject: [PATCH 08/12] comments --- cron/config/config.yaml | 4 ++-- cron/k8s/controller.release.yaml | 4 ++-- cron/k8s/transfer-raw.yaml | 8 ++++---- cron/k8s/transfer.release-raw.yaml | 8 ++++---- cron/k8s/worker.release.yaml | 2 +- cron/worker/main.go | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cron/config/config.yaml b/cron/config/config.yaml index 62ce6ab99ca..bb74415cea5 100644 --- a/cron/config/config.yaml +++ b/cron/config/config.yaml @@ -30,5 +30,5 @@ metric-exporter: stackdriver result-data-bucket-url-v2: gs://ossf-scorecard-data2 bigquery-table-v2: scorecard-v2 # Raw results. -raw-result-data-bucket-url-v0: gs://ossf-scorecard-rawdata-v0 -raw-bigquery-table-v0: scorecard-rawdata-v0 +raw-result-data-bucket-url: gs://ossf-scorecard-rawdata +raw-bigquery-table: scorecard-rawdata diff --git a/cron/k8s/controller.release.yaml b/cron/k8s/controller.release.yaml index c4a5777f78b..7aeb1135e98 100644 --- a/cron/k8s/controller.release.yaml +++ b/cron/k8s/controller.release.yaml @@ -62,9 +62,9 @@ spec: # UPGRADEv2: to remove. - name: SCORECARD_DATA_BUCKET_URLV2 value: "gs://ossf-scorecard-data-releasetest2" - # Raw results v0. + # Raw results. - name: RAW_SCORECARD_DATA_BUCKET_URL - value: "gs://ossf-scorecard-rawdata-releasetest-v0" + value: "gs://ossf-scorecard-rawdata-releasetest" - name: SCORECARD_SHARD_SIZE value: "5" resources: diff --git a/cron/k8s/transfer-raw.yaml b/cron/k8s/transfer-raw.yaml index 7a38f741946..5094e2c2475 100644 --- a/cron/k8s/transfer-raw.yaml +++ b/cron/k8s/transfer-raw.yaml @@ -15,7 +15,7 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: scorecard-bq-raw-transfer-v0 + name: scorecard-bq-raw-transfer spec: # At 02:00UTC on Monday and Thursday. schedule: "0 2 * * 1,4" @@ -25,7 +25,7 @@ spec: template: spec: containers: - - name: bq-raw-transfer-v0 + - name: bq-raw-transfer image: gcr.io/openssf/scorecard-bq-transfer:latest imagePullPolicy: Always resources: @@ -35,7 +35,7 @@ spec: memory: 1Gi env: - name: RAW_SCORECARD_BIGQUERY_TABLE - value: "scorecard-rawdata-v0" + value: "scorecard-rawdata" - name: RAW_SCORECARD_DATA_BUCKET_URL - value: "gs://ossf-scorecard-rawdata-v0" + value: "gs://ossf-scorecard-rawdata" restartPolicy: OnFailure diff --git a/cron/k8s/transfer.release-raw.yaml b/cron/k8s/transfer.release-raw.yaml index 6835a482c9a..1200bb46c72 100644 --- a/cron/k8s/transfer.release-raw.yaml +++ b/cron/k8s/transfer.release-raw.yaml @@ -15,7 +15,7 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: scorecard-bq-transfer-raw-releasetest-v0 + name: scorecard-bq-transfer-raw-releasetest spec: # Daily at 9am and 6pm PST. schedule: "0 1,16 * * *" @@ -26,14 +26,14 @@ spec: spec: restartPolicy: OnFailure containers: - - name: bq-transfer-raw-releasetest-v0 + - name: bq-transfer-raw-releasetest image: gcr.io/openssf/scorecard-bq-transfer:latest imagePullPolicy: Always env: - name: RAW_SCORECARD_DATA_BUCKET_URL - value: "gs://ossf-scorecard-rawdata-releasetest-v0" + value: "gs://ossf-scorecard-rawdata-releasetest" - name: RAW_SCORECARD_BIGQUERY_TABLE - value: "scorecard_raw-releasetest-v0" + value: "scorecard_raw-releasetest" - name: SCORECARD_COMPLETION_THRESHOLD value: "0.9" resources: diff --git a/cron/k8s/worker.release.yaml b/cron/k8s/worker.release.yaml index 78c4ec80bcf..ca02fe8ab17 100644 --- a/cron/k8s/worker.release.yaml +++ b/cron/k8s/worker.release.yaml @@ -38,7 +38,7 @@ spec: - name: SCORECARD_DATA_BUCKET_URLV2 value: "gs://ossf-scorecard-data-releasetest2" - name: RAW_SCORECARD_DATA_BUCKET_URL - value: "gs://ossf-scorecard-rawdata-releasetest-v0" + value: "gs://ossf-scorecard-rawdata-releasetest" - name: SCORECARD_REQUEST_SUBSCRIPTION_URL value: "gcppubsub://projects/openssf/subscriptions/scorecard-batch-worker-releasetest" - name: SCORECARD_BLACKLISTED_CHECKS diff --git a/cron/worker/main.go b/cron/worker/main.go index c66cd3a4e5b..a491da2be68 100644 --- a/cron/worker/main.go +++ b/cron/worker/main.go @@ -138,7 +138,7 @@ func processRequest(ctx context.Context, return fmt.Errorf("error during result.AsJSON2: %w", err) } - // Raw result v0. + // Raw result. if err := format.AsRawJSON(&result, &rawBuffer); err != nil { return fmt.Errorf("error during result.AsRawJSON: %w", err) } @@ -151,7 +151,7 @@ func processRequest(ctx context.Context, return fmt.Errorf("error during WriteToBlobStore2: %w", err) } - // Raw result v0. + // Raw result. if err := data.WriteToBlobStore(ctx, rawBucketURL, filename, rawBuffer.Bytes()); err != nil { return fmt.Errorf("error during WriteToBlobStore2: %w", err) } From dda09c35205b7d62011034f3c11bc74ef408fcd6 Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Fri, 18 Mar 2022 19:44:11 +0000 Subject: [PATCH 09/12] comments --- cron/controller/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron/controller/main.go b/cron/controller/main.go index ebd08f23ec3..ee543700e6a 100644 --- a/cron/controller/main.go +++ b/cron/controller/main.go @@ -161,7 +161,7 @@ func main() { panic(fmt.Errorf("error writing to BlobStore2: %w", err)) } - // Raw data v0. + // Raw data. *metadata.ShardLoc = rawBucket + "/" + data.GetBlobFilename("", t) metadataJSON, err = protojson.Marshal(&metadata) if err != nil { @@ -169,6 +169,6 @@ func main() { } err = data.WriteToBlobStore(ctx, rawBucket, data.GetShardMetadataFilename(t), metadataJSON) if err != nil { - panic(fmt.Errorf("error writing to BlobStore raw v0: %w", err)) + panic(fmt.Errorf("error writing to BlobStore raw: %w", err)) } } From cad61d2f06d5689a32ad94b63cd0500bcbf025bf Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Fri, 18 Mar 2022 19:45:15 +0000 Subject: [PATCH 10/12] comments --- cron/config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron/config/config.go b/cron/config/config.go index d575117790a..baefa266c56 100644 --- a/cron/config/config.go +++ b/cron/config/config.go @@ -50,7 +50,7 @@ const ( // v2 results. bigqueryTableV2 string = "SCORECARD_BIGQUERY_TABLEV2" resultDataBucketURLV2 string = "SCORECARD_DATA_BUCKET_URLV2" - // v0 raw results. + // Raw results. rawBigqueryTable string = "RAW_SCORECARD_BIGQUERY_TABLE" rawResultDataBucketURL string = "RAW_SCORECARD_DATA_BUCKET_URL" ) @@ -81,7 +81,7 @@ type config struct { // UPGRADEv2: to remove. ResultDataBucketURLV2 string `yaml:"result-data-bucket-url-v2"` BigQueryTableV2 string `yaml:"bigquery-table-v2"` - // Raw results v0. + // Raw results. RawResultDataBucketURL string `yaml:"raw-result-data-bucket-url"` RawBigQueryTable string `yaml:"raw-bigquery-table"` } From 2e546043ee5002ac783d6c6e264d3a4f82196d8d Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Fri, 18 Mar 2022 19:45:51 +0000 Subject: [PATCH 11/12] comments --- cron/config/config_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cron/config/config_test.go b/cron/config/config_test.go index be4a43ebaa9..e5800fafee5 100644 --- a/cron/config/config_test.go +++ b/cron/config/config_test.go @@ -39,7 +39,7 @@ const ( // UPGRADEv2: to remove. prodBucketV2 = "gs://ossf-scorecard-data2" prodBigQueryTableV2 = "scorecard-v2" - // v0 raw results. + // Raw results. prodRawBucket = "gs://ossf-scorecard-rawdata" prodRawBigQueryTable = "scorecard-rawdata" ) @@ -76,8 +76,8 @@ func TestYAMLParsing(t *testing.T) { ShardSize: prodShardSize, MetricExporter: prodMetricExporter, // UPGRADEv2: to remove. - ResultDataBucketURLV2: prodBucketV2, - BigQueryTableV2: prodBigQueryTableV2, + ResultDataBucketURLV2: prodBucketV2, + BigQueryTableV2: prodBigQueryTableV2, RawResultDataBucketURL: prodRawBucket, RawBigQueryTable: prodRawBigQueryTable, }, From a4292d970d90af2fff938c3b7c6c298858e5690f Mon Sep 17 00:00:00 2001 From: laurentsimon Date: Fri, 18 Mar 2022 19:59:28 +0000 Subject: [PATCH 12/12] comments --- cron/k8s/transfer-raw.yaml | 2 +- cron/k8s/transfer.release-raw.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cron/k8s/transfer-raw.yaml b/cron/k8s/transfer-raw.yaml index 5094e2c2475..faff1971b15 100644 --- a/cron/k8s/transfer-raw.yaml +++ b/cron/k8s/transfer-raw.yaml @@ -25,7 +25,7 @@ spec: template: spec: containers: - - name: bq-raw-transfer + - name: bq-rawdata-transfer image: gcr.io/openssf/scorecard-bq-transfer:latest imagePullPolicy: Always resources: diff --git a/cron/k8s/transfer.release-raw.yaml b/cron/k8s/transfer.release-raw.yaml index 1200bb46c72..5a30062e272 100644 --- a/cron/k8s/transfer.release-raw.yaml +++ b/cron/k8s/transfer.release-raw.yaml @@ -26,7 +26,7 @@ spec: spec: restartPolicy: OnFailure containers: - - name: bq-transfer-raw-releasetest + - name: bq-transfer-rawdata-releasetest image: gcr.io/openssf/scorecard-bq-transfer:latest imagePullPolicy: Always env: