Add a cron job to copy CII badges data

.gitignore

@@ -7,6 +7,7 @@ cron/data/validate/validate
 cron/data/update/projects-update
 cron/controller/controller
 cron/worker/worker
+cron/cii/cii-worker
 cron/webhook/webhook
 cron/bq/data-transfer
 

Makefile

@@ -93,7 +93,7 @@ tree-status: ## Verify tree is clean and all changes are committed
 
 ################################## make build #################################
 ## Build all cron-related targets
-build-cron: build-pubsub build-bq-transfer build-github-server build-webhook build-add-script \
+build-cron: build-pubsub build-cii-worker build-bq-transfer build-github-server build-webhook build-add-script \
 	  build-validate-script build-update-script
 
 build-targets = generate-mocks generate-docs build-proto build-scorecard build-cron ko-build-everything dockerbuild
@@ -139,6 +139,10 @@ build-pubsub: ## Runs go build on the PubSub cron job
 	cd cron/controller && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o controller
 	cd cron/worker && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o worker
 
+build-cii-worker: ## Runs go build on the CII worker
+	# Run go build on the CII worker
+	cd cron/cii && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o cii-worker
+
 build-bq-transfer: ## Runs go build on the BQ transfer cron job
 build-bq-transfer: ./cron/bq/*.go
 	# Run go build on the Copier cron job
@@ -182,12 +186,17 @@ ko-build-everything: ## ko builds all binaries.
 	ko publish -B --bare --local \
 			   --platform=$(PLATFORM)\
 			   --push=false \
-			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v3/cron/controller 
+			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v3/cron/controller
 	KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-worker
 	ko publish -B --bare --local \
 			   --platform=$(PLATFORM)\
 			   --push=false \
 			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v3/cron/worker
+	KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cii-worker
+	ko publish -B --bare --local \
+			   --platform=$(PLATFORM)\
+			   --push=false \
+			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v3/cron/cii
 	KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-bq-transfer
 	ko publish -B --bare --local \
 			   --platform=$(PLATFORM)\
@@ -209,6 +218,7 @@ dockerbuild: ## Runs docker build
 	DOCKER_BUILDKIT=1 docker build . --file Dockerfile --tag $(IMAGE_NAME)
 	DOCKER_BUILDKIT=1 docker build . --file cron/controller/Dockerfile --tag $(IMAGE_NAME)-batch-controller
 	DOCKER_BUILDKIT=1 docker build . --file cron/worker/Dockerfile --tag $(IMAGE_NAME)-batch-worker
+	DOCKER_BUILDKIT=1 docker build . --file cron/cii/Dockerfile --tag $(IMAGE_NAME)-cii-worker
 	DOCKER_BUILDKIT=1 docker build . --file cron/bq/Dockerfile --tag $(IMAGE_NAME)-bq-transfer
 	DOCKER_BUILDKIT=1 docker build . --file cron/webhook/Dockerfile --tag ${IMAGE_NAME}-webhook
 	DOCKER_BUILDKIT=1 docker build . --file clients/githubrepo/roundtripper/tokens/server/Dockerfile --tag ${IMAGE_NAME}-github-server

cron/cii/Dockerfile

@@ -0,0 +1,29 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang@sha256:3c4de86eec9cbc619cdd72424abd88326ffcf5d813a8338a7743c55e5898734f AS base
+WORKDIR /src
+ENV CGO_ENABLED=0
+COPY go.* ./
+RUN go mod download
+COPY . ./
+
+FROM base AS cii
+ARG TARGETOS
+ARG TARGETARCH
+RUN CGO_ENABLED=0 make build-cii-worker
+
+FROM gcr.io/distroless/base:nonroot@sha256:46d4514c17aca7a68559ee03975983339fc548e6d1014e2d7633f9123f2d3c59
+COPY --from=cii /src/cron/cii/cii-worker cron/cii/cii-worker
+ENTRYPOINT ["cron/cii/cii-worker"]

cron/cii/main.go

@@ -0,0 +1,98 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package main implements the PubSub controller.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/ossf/scorecard/v3/cron/config"
+	"github.com/ossf/scorecard/v3/cron/data"
+)
+
+const ciiBaseURL = "https://bestpractices.coreinfrastructure.org/projects.json"
+
+type ciiPageResp struct {
+	RepoURL    string `json:"repo_url"`
+	BadgeLevel string `json:"badge_level"`
+}
+
+func writeToCIIDataBucket(ctx context.Context, pageResp []ciiPageResp, bucketURL string) error {
+	for _, project := range pageResp {
+		projectURL := strings.TrimPrefix(project.RepoURL, "https://")
+		projectURL = strings.TrimPrefix(projectURL, "http://")
+		fmt.Printf("Writing result for: %s\n", projectURL)
+		if err := data.WriteToBlobStore(ctx, bucketURL,
+			fmt.Sprintf("%s/result.json", projectURL), []byte(project.BadgeLevel)); err != nil {
+			return fmt.Errorf("error during data.WriteToBlobStore: %w", err)
+		}
+	}
+	return nil
+}
+
+func getPage(ctx context.Context, pageNum int) ([]ciiPageResp, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet,
+		fmt.Sprintf("%s?page=%d", ciiBaseURL, pageNum), nil)
+	if err != nil {
+		return nil, fmt.Errorf("error during http.NewRequestWithContext: %w", err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error during http.DefaultClient.Do: %w", err)
+	}
+	defer resp.Body.Close()
+
+	respContent, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error during io.ReadAll: %w", err)
+	}
+
+	ciiResponse := []ciiPageResp{}
+	if err := json.Unmarshal(respContent, &ciiResponse); err != nil {
+		return nil, fmt.Errorf("error during json.Unmarshal: %w", err)
+	}
+	return ciiResponse, nil
+}
+
+func main() {
+	ctx := context.Background()
+	fmt.Println("Starting...")
+
+	ciiDataBucket, err := config.GetCIIDataBucketURL()
+	if err != nil {
+		panic(err)
+	}
+
+	pageNum := 1
+	pageResp, err := getPage(ctx, pageNum)
+	for err == nil && len(pageResp) > 0 {
+		if err := writeToCIIDataBucket(ctx, pageResp, ciiDataBucket); err != nil {
+			panic(err)
+		}
+		pageNum++
+		pageResp, err = getPage(ctx, pageNum)
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Println("Job completed")
+}

cron/cloudbuild/cii.yaml

@@ -0,0 +1,22 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['build', '.',
+  '--build-arg', 'COMMIT_SHA=$COMMIT_SHA',
+  '-t', 'gcr.io/openssf/scorecard-cii-worker:$COMMIT_SHA',
+  '-t', 'gcr.io/openssf/scorecard-cii-worker:latest',
+  '-f', 'cron/cii/Dockerfile']
+images: ['gcr.io/openssf/scorecard-cii-worker']

cron/config/config.go

@@ -45,6 +45,7 @@ const (
 	shardSize              string = "SCORECARD_SHARD_SIZE"
 	webhookURL             string = "SCORECARD_WEBHOOK_URL"
 	metricExporter         string = "SCORECARD_METRIC_EXPORTER"
+	ciiDataBucketURL       string = "SCORECARD_CII_DATA_BUCKET_URL"
 
 	bigqueryTableV2       string = "SCORECARD_BIGQUERY_TABLEV2"
 	resultDataBucketURLV2 string = "SCORECARD_DATA_BUCKET_URLV2"
@@ -69,6 +70,7 @@ type config struct {
 	BigQueryTable          string  `yaml:"bigquery-table"`
 	CompletionThreshold    float32 `yaml:"completion-threshold"`
 	WebhookURL             string  `yaml:"webhook-url"`
+	CIIDataBucketURL       string  `yaml:"cii-data-bucket-url"`
 	MetricExporter         string  `yaml:"metric-exporter"`
 	ShardSize              int     `yaml:"shard-size"`
 	// UPGRADEv2: to remove.
@@ -206,6 +208,15 @@ func GetWebhookURL() (string, error) {
 	return url, nil
 }
 
+// GetCIIDataBucketURL returns the bucket URL where CII data is stored.
+func GetCIIDataBucketURL() (string, error) {
+	url, err := getStringConfigValue(ciiDataBucketURL, configYAML, "CIIDataBucketURL", "cii-data-bucket-url")
+	if err != nil && !errors.Is(err, ErrorEmptyConfigValue) {
+		return url, err
+	}
+	return url, nil
+}
+
 // GetMetricExporter returns the opencensus exporter type.
 func GetMetricExporter() (string, error) {
 	return getStringConfigValue(metricExporter, configYAML, "MetricExporter", "metric-exporter")

cron/config/config.yaml

@@ -21,6 +21,7 @@ bigquery-table: scorecard
 completion-threshold: 0.99
 shard-size: 10
 webhook-url: 
+cii-data-bucket-url: gs://ossf-scorecard-cii-data
 metric-exporter: stackdriver
 # UPGRADEv2: to remove.
 result-data-bucket-url-v2: gs://ossf-scorecard-data2

cron/config/config_test.go

@@ -32,6 +32,7 @@ const (
 	prodBigQueryTable              = "scorecard"
 	prodCompletionThreshold        = 0.99
 	prodWebhookURL                 = ""
+	prodCIIDataBucket              = "gs://ossf-scorecard-cii-data"
 	prodShardSize           int    = 10
 	prodMetricExporter      string = "stackdriver"
 	// UPGRADEv2: to remove.
@@ -66,6 +67,7 @@ func TestYAMLParsing(t *testing.T) {
 				BigQueryTable:          prodBigQueryTable,
 				CompletionThreshold:    prodCompletionThreshold,
 				WebhookURL:             prodWebhookURL,
+				CIIDataBucketURL:       prodCIIDataBucket,
 				ShardSize:              prodShardSize,
 				MetricExporter:         prodMetricExporter,
 				// UPGRADEv2: to remove.

cron/k8s/cii.yaml

@@ -0,0 +1,31 @@
+# Copyright 2021 Security Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: scorecard-cii-worker
+spec:
+  # At 00:00UTC on 1st of every month.
+  schedule: "@monthly"
+  concurrencyPolicy: "Forbid"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: cii-worker
+            image: gcr.io/openssf/scorecard-cii-worker:stable
+            imagePullPolicy: Always

cron/webhook/main.go

@@ -35,6 +35,7 @@ var images = []string{
 	"gcr.io/openssf/scorecard",
 	"gcr.io/openssf/scorecard-batch-controller",
 	"gcr.io/openssf/scorecard-batch-worker",
+	"gcr.io/openssf/scorecard-cii-worker",
 	"gcr.io/openssf/scorecard-bq-transfer",
 	"gcr.io/openssf/scorecard-github-server",
 }

e2e/permissions_test.go

@@ -45,7 +45,7 @@ var _ = Describe("E2E TEST:"+checks.CheckTokenPermissions, func() {
 				Score:         checker.MinResultScore,
 				NumberOfWarn:  1,
 				NumberOfInfo:  2,
-				NumberOfDebug: 4,
+				NumberOfDebug: 5,
 			}
 			result := checks.TokenPermissions(&req)
 			// UPGRADEv2: to remove.