From 4795ba24bdf9cd5e1d167de9ae6d935cd2346f14 Mon Sep 17 00:00:00 2001 From: Azeem Shaikh Date: Mon, 15 Nov 2021 20:00:46 -0800 Subject: [PATCH] Add a cron job to copy CII badges data --- .gitignore | 1 + Makefile | 14 +++++- cron/cii/Dockerfile | 29 +++++++++++ cron/cii/main.go | 98 ++++++++++++++++++++++++++++++++++++++ cron/cloudbuild/cii.yaml | 22 +++++++++ cron/config/config.go | 11 +++++ cron/config/config.yaml | 1 + cron/config/config_test.go | 2 + cron/k8s/cii.yaml | 31 ++++++++++++ cron/webhook/main.go | 1 + e2e/permissions_test.go | 2 +- 11 files changed, 209 insertions(+), 3 deletions(-) create mode 100644 cron/cii/Dockerfile create mode 100644 cron/cii/main.go create mode 100644 cron/cloudbuild/cii.yaml create mode 100644 cron/k8s/cii.yaml diff --git a/.gitignore b/.gitignore index 459c0668d39d..531fde8f77a7 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ cron/data/validate/validate cron/data/update/projects-update cron/controller/controller cron/worker/worker +cron/cii/cii-worker cron/webhook/webhook cron/bq/data-transfer diff --git a/Makefile b/Makefile index 44614dc09ae2..9fde24969a11 100644 --- a/Makefile +++ b/Makefile @@ -93,7 +93,7 @@ tree-status: ## Verify tree is clean and all changes are committed ################################## make build ################################# ## Build all cron-related targets -build-cron: build-pubsub build-bq-transfer build-github-server build-webhook build-add-script \ +build-cron: build-pubsub build-cii-worker build-bq-transfer build-github-server build-webhook build-add-script \ build-validate-script build-update-script build-targets = generate-mocks generate-docs build-proto build-scorecard build-cron ko-build-everything dockerbuild @@ -139,6 +139,10 @@ build-pubsub: ## Runs go build on the PubSub cron job cd cron/controller && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o controller cd cron/worker && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o worker +build-cii-worker: ## Runs go build on the CII worker + # Run go build on the CII worker + cd cron/cii && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o cii-worker + build-bq-transfer: ## Runs go build on the BQ transfer cron job build-bq-transfer: ./cron/bq/*.go # Run go build on the Copier cron job @@ -182,12 +186,17 @@ ko-build-everything: ## ko builds all binaries. ko publish -B --bare --local \ --platform=$(PLATFORM)\ --push=false \ - --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v3/cron/controller + --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v3/cron/controller KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-worker ko publish -B --bare --local \ --platform=$(PLATFORM)\ --push=false \ --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v3/cron/worker + KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cii-worker + ko publish -B --bare --local \ + --platform=$(PLATFORM)\ + --push=false \ + --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v3/cron/cii KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-bq-transfer ko publish -B --bare --local \ --platform=$(PLATFORM)\ @@ -209,6 +218,7 @@ dockerbuild: ## Runs docker build DOCKER_BUILDKIT=1 docker build . --file Dockerfile --tag $(IMAGE_NAME) DOCKER_BUILDKIT=1 docker build . --file cron/controller/Dockerfile --tag $(IMAGE_NAME)-batch-controller DOCKER_BUILDKIT=1 docker build . --file cron/worker/Dockerfile --tag $(IMAGE_NAME)-batch-worker + DOCKER_BUILDKIT=1 docker build . --file cron/cii/Dockerfile --tag $(IMAGE_NAME)-cii-worker DOCKER_BUILDKIT=1 docker build . --file cron/bq/Dockerfile --tag $(IMAGE_NAME)-bq-transfer DOCKER_BUILDKIT=1 docker build . --file cron/webhook/Dockerfile --tag ${IMAGE_NAME}-webhook DOCKER_BUILDKIT=1 docker build . --file clients/githubrepo/roundtripper/tokens/server/Dockerfile --tag ${IMAGE_NAME}-github-server diff --git a/cron/cii/Dockerfile b/cron/cii/Dockerfile new file mode 100644 index 000000000000..6f6ffef10cfa --- /dev/null +++ b/cron/cii/Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2021 Security Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang@sha256:3c4de86eec9cbc619cdd72424abd88326ffcf5d813a8338a7743c55e5898734f AS base +WORKDIR /src +ENV CGO_ENABLED=0 +COPY go.* ./ +RUN go mod download +COPY . ./ + +FROM base AS cii +ARG TARGETOS +ARG TARGETARCH +RUN CGO_ENABLED=0 make build-cii-worker + +FROM gcr.io/distroless/base:nonroot@sha256:46d4514c17aca7a68559ee03975983339fc548e6d1014e2d7633f9123f2d3c59 +COPY --from=cii /src/cron/cii/cii-worker cron/cii/cii-worker +ENTRYPOINT ["cron/cii/cii-worker"] diff --git a/cron/cii/main.go b/cron/cii/main.go new file mode 100644 index 000000000000..62bd543aa823 --- /dev/null +++ b/cron/cii/main.go @@ -0,0 +1,98 @@ +// Copyright 2021 Security Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main implements the PubSub controller. +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/ossf/scorecard/v3/cron/config" + "github.com/ossf/scorecard/v3/cron/data" +) + +const ciiBaseURL = "https://bestpractices.coreinfrastructure.org/projects.json" + +type ciiPageResp struct { + RepoURL string `json:"repo_url"` + BadgeLevel string `json:"badge_level"` +} + +func writeToCIIDataBucket(ctx context.Context, pageResp []ciiPageResp, bucketURL string) error { + for _, project := range pageResp { + projectURL := strings.TrimPrefix(project.RepoURL, "https://") + projectURL = strings.TrimPrefix(projectURL, "http://") + fmt.Printf("Writing result for: %s\n", projectURL) + if err := data.WriteToBlobStore(ctx, bucketURL, + fmt.Sprintf("%s/result.json", projectURL), []byte(project.BadgeLevel)); err != nil { + return fmt.Errorf("error during data.WriteToBlobStore: %w", err) + } + } + return nil +} + +func getPage(ctx context.Context, pageNum int) ([]ciiPageResp, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + fmt.Sprintf("%s?page=%d", ciiBaseURL, pageNum), nil) + if err != nil { + return nil, fmt.Errorf("error during http.NewRequestWithContext: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("error during http.DefaultClient.Do: %w", err) + } + defer resp.Body.Close() + + respContent, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error during io.ReadAll: %w", err) + } + + ciiResponse := []ciiPageResp{} + if err := json.Unmarshal(respContent, &ciiResponse); err != nil { + return nil, fmt.Errorf("error during json.Unmarshal: %w", err) + } + return ciiResponse, nil +} + +func main() { + ctx := context.Background() + fmt.Println("Starting...") + + ciiDataBucket, err := config.GetCIIDataBucketURL() + if err != nil { + panic(err) + } + + pageNum := 1 + pageResp, err := getPage(ctx, pageNum) + for err == nil && len(pageResp) > 0 { + if err := writeToCIIDataBucket(ctx, pageResp, ciiDataBucket); err != nil { + panic(err) + } + pageNum++ + pageResp, err = getPage(ctx, pageNum) + } + if err != nil { + panic(err) + } + + fmt.Println("Job completed") +} diff --git a/cron/cloudbuild/cii.yaml b/cron/cloudbuild/cii.yaml new file mode 100644 index 000000000000..33b4431f97e6 --- /dev/null +++ b/cron/cloudbuild/cii.yaml @@ -0,0 +1,22 @@ +# Copyright 2021 Security Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: +- name: 'gcr.io/cloud-builders/docker' + args: ['build', '.', + '--build-arg', 'COMMIT_SHA=$COMMIT_SHA', + '-t', 'gcr.io/openssf/scorecard-cii-worker:$COMMIT_SHA', + '-t', 'gcr.io/openssf/scorecard-cii-worker:latest', + '-f', 'cron/cii/Dockerfile'] +images: ['gcr.io/openssf/scorecard-cii-worker'] diff --git a/cron/config/config.go b/cron/config/config.go index 1e392385cffe..45f1ee3ccf82 100644 --- a/cron/config/config.go +++ b/cron/config/config.go @@ -45,6 +45,7 @@ const ( shardSize string = "SCORECARD_SHARD_SIZE" webhookURL string = "SCORECARD_WEBHOOK_URL" metricExporter string = "SCORECARD_METRIC_EXPORTER" + ciiDataBucketURL string = "SCORECARD_CII_DATA_BUCKET_URL" bigqueryTableV2 string = "SCORECARD_BIGQUERY_TABLEV2" resultDataBucketURLV2 string = "SCORECARD_DATA_BUCKET_URLV2" @@ -69,6 +70,7 @@ type config struct { BigQueryTable string `yaml:"bigquery-table"` CompletionThreshold float32 `yaml:"completion-threshold"` WebhookURL string `yaml:"webhook-url"` + CIIDataBucketURL string `yaml:"cii-data-bucket-url"` MetricExporter string `yaml:"metric-exporter"` ShardSize int `yaml:"shard-size"` // UPGRADEv2: to remove. @@ -206,6 +208,15 @@ func GetWebhookURL() (string, error) { return url, nil } +// GetCIIDataBucketURL returns the bucket URL where CII data is stored. +func GetCIIDataBucketURL() (string, error) { + url, err := getStringConfigValue(ciiDataBucketURL, configYAML, "CIIDataBucketURL", "cii-data-bucket-url") + if err != nil && !errors.Is(err, ErrorEmptyConfigValue) { + return url, err + } + return url, nil +} + // GetMetricExporter returns the opencensus exporter type. func GetMetricExporter() (string, error) { return getStringConfigValue(metricExporter, configYAML, "MetricExporter", "metric-exporter") diff --git a/cron/config/config.yaml b/cron/config/config.yaml index 5b9d5dcf3fc4..b4645adf17da 100644 --- a/cron/config/config.yaml +++ b/cron/config/config.yaml @@ -21,6 +21,7 @@ bigquery-table: scorecard completion-threshold: 0.99 shard-size: 10 webhook-url: +cii-data-bucket-url: gs://ossf-scorecard-cii-data metric-exporter: stackdriver # UPGRADEv2: to remove. result-data-bucket-url-v2: gs://ossf-scorecard-data2 diff --git a/cron/config/config_test.go b/cron/config/config_test.go index 4569944a0f73..18cd0869c2c0 100644 --- a/cron/config/config_test.go +++ b/cron/config/config_test.go @@ -32,6 +32,7 @@ const ( prodBigQueryTable = "scorecard" prodCompletionThreshold = 0.99 prodWebhookURL = "" + prodCIIDataBucket = "gs://ossf-scorecard-cii-data" prodShardSize int = 10 prodMetricExporter string = "stackdriver" // UPGRADEv2: to remove. @@ -66,6 +67,7 @@ func TestYAMLParsing(t *testing.T) { BigQueryTable: prodBigQueryTable, CompletionThreshold: prodCompletionThreshold, WebhookURL: prodWebhookURL, + CIIDataBucketURL: prodCIIDataBucket, ShardSize: prodShardSize, MetricExporter: prodMetricExporter, // UPGRADEv2: to remove. diff --git a/cron/k8s/cii.yaml b/cron/k8s/cii.yaml new file mode 100644 index 000000000000..831b5e27f505 --- /dev/null +++ b/cron/k8s/cii.yaml @@ -0,0 +1,31 @@ +# Copyright 2021 Security Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: scorecard-cii-worker +spec: + # At 00:00UTC on 1st of every month. + schedule: "@monthly" + concurrencyPolicy: "Forbid" + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: cii-worker + image: gcr.io/openssf/scorecard-cii-worker:stable + imagePullPolicy: Always diff --git a/cron/webhook/main.go b/cron/webhook/main.go index 70162216b4c7..5acc2eb0e75e 100644 --- a/cron/webhook/main.go +++ b/cron/webhook/main.go @@ -35,6 +35,7 @@ var images = []string{ "gcr.io/openssf/scorecard", "gcr.io/openssf/scorecard-batch-controller", "gcr.io/openssf/scorecard-batch-worker", + "gcr.io/openssf/scorecard-cii-worker", "gcr.io/openssf/scorecard-bq-transfer", "gcr.io/openssf/scorecard-github-server", } diff --git a/e2e/permissions_test.go b/e2e/permissions_test.go index 34b1dd9d8ead..e31e5aa871fe 100644 --- a/e2e/permissions_test.go +++ b/e2e/permissions_test.go @@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() { Score: checker.MinResultScore, NumberOfWarn: 1, NumberOfInfo: 2, - NumberOfDebug: 4, + NumberOfDebug: 5, } result := checks.TokenPermissions(&req) // UPGRADEv2: to remove.