release scripts (#411)

.gitignore

@@ -30,3 +30,7 @@ vendor/sigs.k8s.io/cluster-api/docs/book/*.json
 
 # tilt
 .tiltbuild
+
+# release
+_artifacts/
+out/

Makefile

@@ -12,22 +12,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+ROOT_DIR_RELATIVE := .
+
+include $(ROOT_DIR_RELATIVE)/common.mk
+
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:crdVersions=v1"
+
+# Directories.
+REPO_ROOT := $(shell git rev-parse --show-toplevel)
+ARTIFACTS ?= $(REPO_ROOT)/_artifacts
 TOOLS_DIR := hack/tools
-TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/bin)
+TOOLS_BIN_DIR := $(TOOLS_DIR)/bin
 GO_INSTALL = ./scripts/go_install.sh
-GOLANGCI_LINT_VER := v1.31.0
-GOLANGCI_LINT_BIN := golangci-lint
-GOLANGCI_LINT := $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER)
+
+GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint
+KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
+GOJQ := $(TOOLS_BIN_DIR)/gojq
 
 STAGING_REGISTRY ?= gcr.io/k8s-staging-capi-ibmcloud
+PROD_REGISTRY := k8s.gcr.io/capi-ibmcloud
 REGISTRY ?= $(STAGING_REGISTRY)
 RELEASE_TAG ?= $(shell git describe --abbrev=0 2>/dev/null)
 PULL_BASE_REF ?= $(RELEASE_TAG) # PULL_BASE_REF will be provided by Prow
 RELEASE_ALIAS_TAG ?= $(PULL_BASE_REF)
+RELEASE_DIR := out
 
 TAG ?= dev
 ARCH ?= amd64
@@ -36,6 +47,11 @@ ALL_ARCH ?= amd64 ppc64le
 # main controller
 CORE_IMAGE_NAME ?= cluster-api-ibmcloud-controller
 CORE_CONTROLLER_IMG ?= $(REGISTRY)/$(CORE_IMAGE_NAME)
+CORE_CONTROLLER_ORIGINAL_IMG := gcr.io/k8s-staging-capi-ibmcloud/cluster-api-ibmcloud-controller
+CORE_CONTROLLER_NAME := controller-manager
+CORE_MANIFEST_FILE := infrastructure-components
+CORE_CONFIG_DIR := config/default
+CORE_NAMESPACE := capi-ibmcloud-system
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -113,9 +129,6 @@ endif
 lint: $(GOLANGCI_LINT) ## Lint codebase
 	$(GOLANGCI_LINT) run -v --fast=false
 
-$(GOLANGCI_LINT): ## Build golangci-lint from tools folder.
-	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER)
-
 ## --------------------------------------
 ## Docker
 ## --------------------------------------
@@ -165,15 +178,126 @@ docker-push-manifest:
 ## Release
 ## --------------------------------------
 
-.PHONY: release-alias-tag
-release-alias-tag: # Adds the tag to the last build tag.
-	gcloud container images add-tag -q $(CORE_CONTROLLER_IMG):$(TAG) $(CORE_CONTROLLER_IMG):$(RELEASE_ALIAS_TAG)
+$(RELEASE_DIR):
+	mkdir -p $@
+
+$(ARTIFACTS):
+	mkdir -p $@
+
+.PHONY: list-staging-releases
+list-staging-releases: ## List staging images for image promotion
+	@echo $(CORE_IMAGE_NAME):
+	$(MAKE) list-image RELEASE_TAG=$(RELEASE_TAG) IMAGE=$(CORE_IMAGE_NAME)
+
+list-image:
+	gcloud container images list-tags $(STAGING_REGISTRY)/$(IMAGE) --filter="tags=('$(RELEASE_TAG)')" --format=json
+
+.PHONY: check-release-tag
+check-release-tag:
+	@if [ -z "${RELEASE_TAG}" ]; then echo "RELEASE_TAG is not set"; exit 1; fi
+	@if ! [ -z "$$(git status --porcelain)" ]; then echo "Your local git repository contains uncommitted changes, use git clean before proceeding."; exit 1; fi
+
+.PHONY: check-previous-release-tag
+check-previous-release-tag:
+	@if [ -z "${PREVIOUS_VERSION}" ]; then echo "PREVIOUS_VERSION is not set"; exit 1; fi
+
+.PHONY: check-github-token
+check-github-token:
+	@if [ -z "${GITHUB_TOKEN}" ]; then echo "GITHUB_TOKEN is not set"; exit 1; fi
+
+.PHONY: release
+release: clean-release check-release-tag $(RELEASE_DIR)  ## Builds and push container images using the latest git tag for the commit.
+	git checkout "${RELEASE_TAG}"
+	CORE_CONTROLLER_IMG=$(PROD_REGISTRY)/$(CORE_IMAGE_NAME) $(MAKE) release-manifests
+	$(MAKE) release-templates
+
+.PHONY: release-manifests
+release-manifests:
+	$(MAKE) $(RELEASE_DIR)/$(CORE_MANIFEST_FILE).yaml TAG=$(RELEASE_TAG)
+	# Add metadata to the release artifacts
+	cp metadata.yaml $(RELEASE_DIR)/metadata.yaml
 
 .PHONY: release-staging
-release-staging: ## Builds and push container images to the staging image registry.
+release-staging: ## Builds and push container images and manifests to the staging bucket.
 	$(MAKE) docker-build-all
 	$(MAKE) docker-push-all
 	$(MAKE) release-alias-tag
+	$(MAKE) staging-manifests
+	$(MAKE) release-templates
+	$(MAKE) upload-staging-artifacts
+
+.PHONY: staging-manifests
+staging-manifests:
+	$(MAKE) $(RELEASE_DIR)/$(CORE_MANIFEST_FILE).yaml TAG=$(RELEASE_ALIAS_TAG)
+	cp metadata.yaml $(RELEASE_DIR)/metadata.yaml
+
+.PHONY: upload-staging-artifacts
+upload-staging-artifacts: ## Upload release artifacts to the staging bucket
+	gsutil cp $(RELEASE_DIR)/* gs://$(BUCKET)/components/$(RELEASE_ALIAS_TAG)
+
+.PHONY: release-alias-tag
+release-alias-tag: # Adds the tag to the last build tag.
+	gcloud container images add-tag -q $(CORE_CONTROLLER_IMG):$(TAG) $(CORE_CONTROLLER_IMG):$(RELEASE_ALIAS_TAG)
+
+.PHONY: release-templates
+release-templates: $(RELEASE_DIR) ## Generate release templates
+	cp templates/cluster-template*.yaml $(RELEASE_DIR)/
+
+IMAGE_PATCH_DIR := $(ARTIFACTS)/image-patch
+
+$(IMAGE_PATCH_DIR): $(ARTIFACTS)
+	mkdir -p $@
+
+.PHONY: $(RELEASE_DIR)/$(CORE_MANIFEST_FILE).yaml
+$(RELEASE_DIR)/$(CORE_MANIFEST_FILE).yaml:
+	$(MAKE) compiled-manifest \
+		PROVIDER=$(CORE_MANIFEST_FILE) \
+		OLD_IMG=$(CORE_CONTROLLER_ORIGINAL_IMG) \
+		MANIFEST_IMG=$(CORE_CONTROLLER_IMG) \
+		CONTROLLER_NAME=$(CORE_CONTROLLER_NAME) \
+		PROVIDER_CONFIG_DIR=$(CORE_CONFIG_DIR) \
+		NAMESPACE=$(CORE_NAMESPACE) \
+
+.PHONY: compiled-manifest
+compiled-manifest: $(RELEASE_DIR) $(KUSTOMIZE)
+	$(MAKE) image-patch-source-manifest
+	$(MAKE) image-patch-kustomization
+	$(KUSTOMIZE) build $(IMAGE_PATCH_DIR)/$(PROVIDER) > $(RELEASE_DIR)/$(PROVIDER).yaml
+
+.PHONY: image-patch-source-manifest
+image-patch-source-manifest: $(IMAGE_PATCH_DIR) $(KUSTOMIZE)
+	mkdir -p $(IMAGE_PATCH_DIR)/$(PROVIDER)
+	$(KUSTOMIZE) build $(PROVIDER_CONFIG_DIR) > $(IMAGE_PATCH_DIR)/$(PROVIDER)/source-manifest.yaml
+
+.PHONY: image-patch-kustomization
+image-patch-kustomization: $(IMAGE_PATCH_DIR)
+	mkdir -p $(IMAGE_PATCH_DIR)/$(PROVIDER)
+	$(MAKE) image-patch-kustomization-without-webhook
+
+.PHONY: image-patch-kustomization-without-webhook
+image-patch-kustomization-without-webhook: $(IMAGE_PATCH_DIR) $(GOJQ)
+	mkdir -p $(IMAGE_PATCH_DIR)/$(PROVIDER)
+	$(GOJQ) --yaml-input --yaml-output '.images[0]={"name":"$(OLD_IMG)","newName":"$(MANIFEST_IMG)","newTag":"$(TAG)"}' \
+		"hack/image-patch/kustomization.yaml" > $(IMAGE_PATCH_DIR)/$(PROVIDER)/kustomization.yaml
+
+## --------------------------------------
+## Cleanup / Verification
+## --------------------------------------
+
+.PHONY: clean
+clean: ## Remove all generated files
+	$(MAKE) -C hack/tools clean
+	$(MAKE) clean-temporary
+
+.PHONY: clean-temporary
+clean-temporary: ## Remove all temporary files and folders
+	rm -f minikube.kubeconfig
+	rm -f kubeconfig
+	rm -rf _artifacts
+
+.PHONY: clean-release
+clean-release: ## Remove the release folder
+	rm -rf $(RELEASE_DIR)
 
 .PHONY: verify
 verify:

common.mk

@@ -0,0 +1,8 @@
+
+
+TOOLS_DIR := $(ROOT_DIR_RELATIVE)/hack/tools
+TOOLS_DIR_DEPS := $(TOOLS_DIR)/go.sum $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/Makefile
+TOOLS_BIN_DIR := $(TOOLS_DIR)/bin
+
+$(TOOLS_BIN_DIR)/%: $(TOOLS_DIR_DEPS)
+	make -C $(TOOLS_DIR) $(subst $(TOOLS_DIR)/,,$@)

hack/image-patch/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+images:
+  - name: ""
+    newName: ""
+    newTag: ""
+kind: Kustomization
+resources:
+  - source-manifest.yaml

hack/tools/Makefile

@@ -0,0 +1,31 @@
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Directories.
+BIN_DIR := bin
+
+$(BIN_DIR):
+	mkdir -p $@
+
+GOLANGCI_LINT := $(BIN_DIR)/golangci-lint
+$(GOLANGCI_LINT): $(BIN_DIR) go.mod go.sum # Build golangci-lint from tools folder.
+	go build -tags=tools -o $@ github.com/golangci/golangci-lint/cmd/golangci-lint
+
+KUSTOMIZE := $(BIN_DIR)/kustomize
+$(KUSTOMIZE): $(BIN_DIR) go.mod go.sum # Build kustomize from tools folder.
+	CGO_ENABLED=0 go build -tags=tools -o $@ sigs.k8s.io/kustomize/kustomize/v3
+
+GOJQ := $(BIN_DIR)/gojq
+$(GOJQ): $(BIN_DIR) go.mod go.sum
+	go build -tags=tools -o $@ github.com/itchyny/gojq/cmd/gojq

hack/tools/go.mod

@@ -5,7 +5,9 @@ go 1.16
 require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/drone/envsubst/v2 v2.0.0-20210615175204-7bf45dbf5372
-	github.com/hashicorp/go-multierror v1.0.0
+	github.com/golangci/golangci-lint v1.41.1
+	github.com/hashicorp/go-multierror v1.1.1
+	github.com/itchyny/gojq v0.12.5
 	github.com/joelanford/go-apidiff v0.1.0
 	github.com/onsi/ginkgo v1.16.4
 	github.com/pkg/errors v0.9.1
@@ -18,4 +20,5 @@ require (
 	sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20210827150604-1730628f118b
 	sigs.k8s.io/controller-tools v0.7.0
 	sigs.k8s.io/kubebuilder/docs/book/utils v0.0.0-20210702145813-742983631190
+	sigs.k8s.io/kustomize/kustomize/v3 v3.10.0
 )