Merge branch 'ossf:main' into issue1908

.gitignore

@@ -2,15 +2,15 @@
 scorecard
 gitblobcache
 clients/githubrepo/roundtripper/tokens/server/github-auth-server
-cron/data/add/add
-cron/data/validate/validate
-cron/data/update/projects-update
-cron/controller/controller
-cron/worker/worker
-cron/cii/cii-worker
-cron/shuffle/shuffle
-cron/webhook/webhook
-cron/bq/data-transfer
+cron/internal/data/add/add
+cron/internal/data/validate/validate
+cron/internal/data/update/projects-update
+cron/internal/controller/controller
+cron/internal/worker/worker
+cron/internal/cii/cii-worker
+cron/internal/shuffle/shuffle
+cron/internal/webhook/webhook
+cron/internal/bq/data-transfer
 
 # Binaries for programs and plugins.
 *.exe

Makefile

@@ -74,16 +74,16 @@ check-osv: $(install)
 	go list -m -f '{{if not (or  .Main)}}{{.Path}}@{{.Version}}_{{.Replace}}{{end}}' all \
 			| stunning-tribble 
 
-add-projects: ## Adds new projects to ./cron/data/projects.csv
-add-projects: ./cron/data/projects.csv | build-add-script
-	# Add new projects to ./cron/data/projects.csv
-	./cron/data/add/add ./cron/data/projects.csv ./cron/data/projects.new.csv
-	mv ./cron/data/projects.new.csv ./cron/data/projects.csv
+add-projects: ## Adds new projects to ./cron/internal/data/projects.csv
+add-projects: ./cron/internal/data/projects.csv | build-add-script
+	# Add new projects to ./cron/internal/data/projects.csv
+	./cron/internal/data/add/add ./cron/internal/data/projects.csv ./cron/internal/data/projects.new.csv
+	mv ./cron/internal/data/projects.new.csv ./cron/internal/data/projects.csv
 
-validate-projects: ## Validates ./cron/data/projects.csv
-validate-projects: ./cron/data/projects.csv | build-validate-script
-	# Validate ./cron/data/projects.csv
-	./cron/data/validate/validate ./cron/data/projects.csv
+validate-projects: ## Validates ./cron/internal/data/projects.csv
+validate-projects: ./cron/internal/data/projects.csv | build-validate-script
+	# Validate ./cron/internal/data/projects.csv
+	./cron/internal/data/validate/validate ./cron/internal/data/projects.csv
 
 tree-status: | all-targets-update-dependencies ## Verify tree is clean and all changes are committed
 	# Verify the tree is clean and all changes are commited
@@ -103,11 +103,11 @@ build: ## Build all binaries and images in the repo.
 build: $(build-targets)
 
 build-proto: ## Compiles and generates all required protobufs
-build-proto: cron/data/request.pb.go cron/data/metadata.pb.go
-cron/data/request.pb.go: cron/data/request.proto |  $(PROTOC)
-	protoc --go_out=../../../ cron/data/request.proto
-cron/data/metadata.pb.go: cron/data/metadata.proto |  $(PROTOC)
-	protoc --go_out=../../../ cron/data/metadata.proto
+build-proto: cron/internal/data/request.pb.go cron/internal/data/metadata.pb.go
+cron/internal/data/request.pb.go: cron/internal/data/request.proto |  $(PROTOC)
+	protoc --go_out=../../../ cron/internal/data/request.proto
+cron/internal/data/metadata.pb.go: cron/internal/data/metadata.proto |  $(PROTOC)
+	protoc --go_out=../../../ cron/internal/data/metadata.proto
 
 generate-mocks: ## Compiles and generates all mocks using mockgen.
 generate-mocks: clients/mockclients/repo_client.go clients/mockclients/repo.go clients/mockclients/cii_client.go checks/mockclients/vulnerabilities.go cmd/packagemanager_mockclient.go
@@ -147,24 +147,24 @@ build-releaser: ## Runs goreleaser on the repo
 
 build-controller: ## Runs go build on the cron PubSub controller
 	# Run go build on the cron PubSub controller
-	cd cron/controller && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o controller
+	cd cron/internal/controller && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o controller
 
 build-worker: ## Runs go build on the cron PubSub worker
 	# Run go build on the cron PubSub worker
-	cd cron/worker && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o worker
+	cd cron/internal/worker && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o worker
 
 build-cii-worker: ## Runs go build on the CII worker
 	# Run go build on the CII worker
-	cd cron/cii && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o cii-worker
+	cd cron/internal/cii && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o cii-worker
 
 build-shuffler: ## Runs go build on the cron shuffle script
 	# Run go build on the cron shuffle script
-	cd cron/shuffle && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o shuffle
+	cd cron/internal/shuffle && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o shuffle
 
 build-bq-transfer: ## Runs go build on the BQ transfer cron job
-build-bq-transfer: ./cron/bq/*.go
+build-bq-transfer: ./cron/internal/bq/*.go
 	# Run go build on the Copier cron job
-	cd cron/bq && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o data-transfer
+	cd cron/internal/bq && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o data-transfer
 
 build-github-server: ## Runs go build on the GitHub auth server
 build-github-server: ./clients/githubrepo/roundtripper/tokens/*
@@ -174,25 +174,25 @@ build-github-server: ./clients/githubrepo/roundtripper/tokens/*
 
 build-webhook: ## Runs go build on the cron webhook
 	# Run go build on the cron webhook
-	cd cron/webhook && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o webhook
+	cd cron/internal/webhook && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o webhook
 
 build-add-script: ## Runs go build on the add script
-build-add-script: cron/data/add/add
-cron/data/add/add: cron/data/add/*.go cron/data/*.go cron/data/projects.csv
+build-add-script: cron/internal/data/add/add
+cron/internal/data/add/add: cron/internal/data/add/*.go cron/internal/data/*.go cron/internal/data/projects.csv
 	# Run go build on the add script
-	cd cron/data/add && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o add
+	cd cron/internal/data/add && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o add
 
 build-validate-script: ## Runs go build on the validate script
-build-validate-script: cron/data/validate/validate
-cron/data/validate/validate: cron/data/validate/*.go cron/data/*.go cron/data/projects.csv
+build-validate-script: cron/internal/data/validate/validate
+cron/internal/data/validate/validate: cron/internal/data/validate/*.go cron/internal/data/*.go cron/internal/data/projects.csv
 	# Run go build on the validate script
-	cd cron/data/validate && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o validate
+	cd cron/internal/data/validate && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o validate
 
 build-update-script: ## Runs go build on the update script
-build-update-script: cron/data/update/projects-update
-cron/data/update/projects-update:  cron/data/update/*.go cron/data/*.go
+build-update-script: cron/internal/data/update/projects-update
+cron/internal/data/update/projects-update:  cron/internal/data/update/*.go cron/internal/data/*.go
 	# Run go build on the update script
-	cd cron/data/update && CGO_ENABLED=0 go build -trimpath -a -tags netgo -ldflags '$(LDFLAGS)'  -o projects-update
+	cd cron/internal/data/update && CGO_ENABLED=0 go build -trimpath -a -tags netgo -ldflags '$(LDFLAGS)'  -o projects-update
 
 ko-targets = scorecard-ko cron-controller-ko cron-worker-ko cron-cii-worker-ko cron-bq-transfer-ko cron-webhook-ko cron-github-server-ko
 .PHONY: ko-build-everything $(ko-targets)
@@ -212,39 +212,39 @@ cron-controller-ko:
 			   --push=false \
 			   --sbom=none \
 			   --platform=$(PLATFORM)\
-			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/controller
+			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/controller
 cron-worker-ko:
 	$(call_create_kocache_path)
 	KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-batch-worker LDFLAGS="$(LDFLAGS)" \
 	KOCACHE=$(KOCACHE_PATH) ko build -B \
 			   --push=false \
 			   --sbom=none \
 			   --platform=$(PLATFORM)\
-			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/worker
+			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/worker
 cron-cii-worker-ko:
 	$(call_create_kocache_path)
 	KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cii-worker LDFLAGS="$(LDFLAGS)" \
 	KOCACHE=$(KOCACHE_PATH) ko build -B \
 			   --push=false \
 			   --sbom=none \
 			   --platform=$(PLATFORM)\
-			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/cii
+			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/cii
 cron-bq-transfer-ko:
 	$(call_create_kocache_path)
 	KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-bq-transfer LDFLAGS="$(LDFLAGS)" \
 	KOCACHE=$(KOCACHE_PATH) ko build -B \
 			   --push=false \
 			   --sbom=none \
 			   --platform=$(PLATFORM)\
-			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/bq
+			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/bq
 cron-webhook-ko:
 	$(call_create_kocache_path)
 	KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-cron-webhook LDFLAGS="$(LDFLAGS)" \
 	KOCACHE=$(KOCACHE_PATH) ko build -B \
 			   --push=false \
 			   --sbom=none \
 			   --platform=$(PLATFORM)\
-			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/webhook
+			   --tags latest,$(GIT_VERSION),$(GIT_HASH) github.com/ossf/scorecard/v4/cron/internal/webhook
 cron-github-server-ko:
 	$(call_create_kocache_path)
 	KO_DATA_DATE_EPOCH=$(SOURCE_DATE_EPOCH) KO_DOCKER_REPO=${KO_PREFIX}/$(IMAGE_NAME)-github-server LDFLAGS="$(LDFLAGS)" \
@@ -261,15 +261,15 @@ dockerbuild: $(docker-targets)
 scorecard-docker:
 	DOCKER_BUILDKIT=1 docker build . --file Dockerfile --tag $(IMAGE_NAME)
 cron-controller-docker:
-	DOCKER_BUILDKIT=1 docker build . --file cron/controller/Dockerfile --tag $(IMAGE_NAME)-batch-controller
+	DOCKER_BUILDKIT=1 docker build . --file cron/internal/controller/Dockerfile --tag $(IMAGE_NAME)-batch-controller
 cron-worker-docker:
-	DOCKER_BUILDKIT=1 docker build . --file cron/worker/Dockerfile --tag $(IMAGE_NAME)-batch-worker
+	DOCKER_BUILDKIT=1 docker build . --file cron/internal/worker/Dockerfile --tag $(IMAGE_NAME)-batch-worker
 cron-cii-worker-docker:
-	DOCKER_BUILDKIT=1 docker build . --file cron/cii/Dockerfile --tag $(IMAGE_NAME)-cii-worker
+	DOCKER_BUILDKIT=1 docker build . --file cron/internal/cii/Dockerfile --tag $(IMAGE_NAME)-cii-worker
 cron-bq-transfer-docker:
-	DOCKER_BUILDKIT=1 docker build . --file cron/bq/Dockerfile --tag $(IMAGE_NAME)-bq-transfer
+	DOCKER_BUILDKIT=1 docker build . --file cron/internal/bq/Dockerfile --tag $(IMAGE_NAME)-bq-transfer
 cron-webhook-docker:
-	DOCKER_BUILDKIT=1 docker build . --file cron/webhook/Dockerfile --tag ${IMAGE_NAME}-webhook
+	DOCKER_BUILDKIT=1 docker build . --file cron/internal/webhook/Dockerfile --tag ${IMAGE_NAME}-webhook
 cron-github-server-docker:
 	DOCKER_BUILDKIT=1 docker build . --file clients/githubrepo/roundtripper/tokens/server/Dockerfile --tag ${IMAGE_NAME}-github-server
 ###############################################################################

cron/cloudbuild/cii.yaml

@@ -18,5 +18,5 @@ steps:
   '--build-arg', 'COMMIT_SHA=$COMMIT_SHA',
   '-t', 'gcr.io/openssf/scorecard-cii-worker:$COMMIT_SHA',
   '-t', 'gcr.io/openssf/scorecard-cii-worker:latest',
-  '-f', 'cron/cii/Dockerfile']
+  '-f', 'cron/internal/cii/Dockerfile']
 images: ['gcr.io/openssf/scorecard-cii-worker']

cron/cloudbuild/controller.yaml

@@ -18,5 +18,5 @@ steps:
   '--build-arg', 'COMMIT_SHA=$COMMIT_SHA',
   '-t', 'gcr.io/openssf/scorecard-batch-controller:$COMMIT_SHA',
   '-t', 'gcr.io/openssf/scorecard-batch-controller:latest',
-  '-f', 'cron/controller/Dockerfile']
+  '-f', 'cron/internal/controller/Dockerfile']
 images: ['gcr.io/openssf/scorecard-batch-controller']

cron/cloudbuild/transfer.yaml

@@ -17,5 +17,5 @@ steps:
   args: ['build', '.',
   '-t', 'gcr.io/openssf/scorecard-bq-transfer:$COMMIT_SHA',
   '-t', 'gcr.io/openssf/scorecard-bq-transfer:latest',
-  '-f', 'cron/bq/Dockerfile']
+  '-f', 'cron/internal/bq/Dockerfile']
 images: ['gcr.io/openssf/scorecard-bq-transfer']

cron/cloudbuild/webhook.release.yaml

@@ -17,5 +17,5 @@ steps:
   args: ['build', '.',
   '-t', 'gcr.io/openssf/scorecard-webhook-releasetest:$COMMIT_SHA',
   '-t', 'gcr.io/openssf/scorecard-webhook-releasetest:latest',
-  '-f', 'cron/webhook/Dockerfile']
+  '-f', 'cron/internal/webhook/Dockerfile']
 images: ['gcr.io/openssf/scorecard-webhook-releasetest']

cron/cloudbuild/worker.yaml

@@ -20,5 +20,5 @@ steps:
   args: ['build', '.',
   '-t', 'gcr.io/openssf/scorecard-batch-worker:$COMMIT_SHA',
   '-t', 'gcr.io/openssf/scorecard-batch-worker:latest',
-  '-f', 'cron/worker/Dockerfile']
+  '-f', 'cron/internal/worker/Dockerfile']
 images: ['gcr.io/openssf/scorecard-batch-worker']

cron/bq/Dockerfile → cron/internal/bq/Dockerfile

@@ -25,5 +25,5 @@ ARG TARGETARCH
 RUN CGO_ENABLED=0 make build-bq-transfer
 
 FROM gcr.io/distroless/base:nonroot@sha256:19d927c16ddb5415d5f6f529dbbeb13c460b84b304b97af886998d3fcf18ac81
-COPY --from=transfer /src/cron/bq/data-transfer cron/bq/data-transfer
-ENTRYPOINT ["cron/bq/data-transfer"]
+COPY --from=transfer /src/cron/internal/bq/data-transfer cron/internal/bq/data-transfer
+ENTRYPOINT ["cron/internal/bq/data-transfer"]

cron/bq/main.go → cron/internal/bq/main.go

@@ -27,8 +27,8 @@ import (
 
 	"google.golang.org/protobuf/encoding/protojson"
 
-	"github.com/ossf/scorecard/v4/cron/config"
-	"github.com/ossf/scorecard/v4/cron/data"
+	"github.com/ossf/scorecard/v4/cron/internal/config"
+	"github.com/ossf/scorecard/v4/cron/internal/data"
 )
 
 type shardSummary struct {

cron/cii/Dockerfile → cron/internal/cii/Dockerfile

@@ -25,5 +25,5 @@ ARG TARGETARCH
 RUN CGO_ENABLED=0 make build-cii-worker
 
 FROM gcr.io/distroless/base:nonroot@sha256:46d4514c17aca7a68559ee03975983339fc548e6d1014e2d7633f9123f2d3c59
-COPY --from=cii /src/cron/cii/cii-worker cron/cii/cii-worker
-ENTRYPOINT ["cron/cii/cii-worker"]
+COPY --from=cii /src/cron/internal/cii/cii-worker cron/internal/cii/cii-worker
+ENTRYPOINT ["cron/internal/cii/cii-worker"]

cron/cii/main.go → cron/internal/cii/main.go

@@ -24,8 +24,8 @@ import (
 	"strings"
 
 	"github.com/ossf/scorecard/v4/clients"
-	"github.com/ossf/scorecard/v4/cron/config"
-	"github.com/ossf/scorecard/v4/cron/data"
+	"github.com/ossf/scorecard/v4/cron/internal/config"
+	"github.com/ossf/scorecard/v4/cron/internal/data"
 )
 
 const ciiBaseURL = "https://bestpractices.coreinfrastructure.org/projects.json"

cron/controller/Dockerfile → cron/internal/controller/Dockerfile

@@ -23,15 +23,15 @@ FROM base AS shuffle
 ARG TARGETOS
 ARG TARGETARCH
 RUN CGO_ENABLED=0 make build-shuffler
-RUN ./cron/shuffle/shuffle 2500 cron/data/projects.csv cron/data/projects.release.csv
+RUN ./cron/internal/shuffle/shuffle 2500 cron/internal/data/projects.csv cron/internal/data/projects.release.csv
 
 FROM base AS controller
 ARG TARGETOS
 ARG TARGETARCH
 RUN CGO_ENABLED=0 make build-controller
 
 FROM gcr.io/distroless/base:nonroot@sha256:d65ac1a65a4d82a48ebd0a22aea2acdd95d7abeeda245dfee932ec0018c781f4
-COPY ./cron/data/projects*csv cron/data/
-COPY --from=shuffle /src/cron/data/projects.release.csv cron/data/projects.release.csv
-COPY --from=controller /src/cron/controller/controller cron/controller/controller
-ENTRYPOINT ["cron/controller/controller"]
+COPY ./cron/internal/data/projects*csv cron/data/
+COPY --from=shuffle /src/cron/internal/data/projects.release.csv cron/internal/data/projects.release.csv
+COPY --from=controller /src/cron/internal/controller/controller cron/internal/controller/controller
+ENTRYPOINT ["cron/internal/controller/controller"]

cron/controller/main.go → cron/internal/controller/main.go

@@ -26,9 +26,9 @@ import (
 	"sigs.k8s.io/release-utils/version"
 
 	"github.com/ossf/scorecard/v4/clients"
-	"github.com/ossf/scorecard/v4/cron/config"
-	"github.com/ossf/scorecard/v4/cron/data"
-	"github.com/ossf/scorecard/v4/cron/pubsub"
+	"github.com/ossf/scorecard/v4/cron/internal/config"
+	"github.com/ossf/scorecard/v4/cron/internal/data"
+	"github.com/ossf/scorecard/v4/cron/internal/pubsub"
 )
 
 var headSHA = clients.HeadSHA

cron/data/add/main.go → cron/internal/data/add/main.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/ossf/scorecard/v4/cron/data"
+	"github.com/ossf/scorecard/v4/cron/internal/data"
 )
 
 // Script to add new project repositories to the projects.csv file:

cron/data/add/main_test.go → cron/internal/data/add/main_test.go

@@ -22,7 +22,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 
-	"github.com/ossf/scorecard/v4/cron/data"
+	"github.com/ossf/scorecard/v4/cron/internal/data"
 )
 
 func lessThanURI(x, y data.RepoFormat) bool {

cron/data/blob.go → cron/internal/data/blob.go

@@ -26,7 +26,7 @@ import (
 	// Needed to link in GCP drivers.
 	_ "gocloud.dev/blob/gcsblob"
 
-	"github.com/ossf/scorecard/v4/cron/config"
+	"github.com/ossf/scorecard/v4/cron/internal/config"
 )
 
 const (