🌱 Add a way to run ETCDRestore controller locally

.github/workflows/nightly-chart-and-image-publish.yaml

@@ -32,7 +32,9 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
     - name: Build and push docker images
-      run: make docker-build-and-push TAG=${{ env.TAG }} ORG=${{ env.PROD_ORG }}
+      run: |
+        make docker-build-and-push TAG=${{ env.TAG }} ORG=${{ env.PROD_ORG }}
+        make docker-build-and-push-etcdrestore TAG=${{ env.TAG }} ORG=${{ env.PROD_ORG }}
 
   publish-helm-chart-ghcr:
     name: Publish Helm chart to GHCR

Makefile

@@ -169,6 +169,10 @@ CONTROLLER_IMG ?= $(REGISTRY)/$(ORG)/$(CONTROLLER_IMAGE_NAME)
 CONTROLLER_IMAGE_VERSION ?= $(shell git describe --abbrev=0 2>/dev/null)
 IID_FILE ?= $(shell mktemp)
 
+# etcdrestore
+ETCDRESTORE_IMAGE_NAME ?= turtles-etcd-snapshot-restore
+ETCDRESTORE_IMG ?= $(REGISTRY)/$(ORG)/$(ETCDRESTORE_IMAGE_NAME)
+
 # Release
 # Exclude tags with the prefix 'test/'
 RELEASE_TAG ?= $(shell git describe --abbrev=0 --exclude 'test/*' 2>/dev/null)
@@ -341,6 +345,32 @@ docker-pull-prerequisites:
 	docker pull $(GO_CONTAINER_IMAGE)
 	docker pull gcr.io/distroless/static:latest
 
+.PHONY: docker-build-etcdrestore ## Build the docker image for etcdrestore
+docker-build-etcdrestore: buildx-machine docker-pull-prerequisites ## Build docker image for a specific architecture
+## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368)
+	# buildx does not support using local registry for multi-architecture images
+	cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build \
+			--platform $(ARCH) \
+			--load \
+			--build-arg builder_image=$(GO_CONTAINER_IMAGE) \
+			--build-arg goproxy=$(GOPROXY) \
+			--build-arg package=./exp/etcdrestore \
+			--build-arg ldflags="$(LDFLAGS)" . -t $(ETCDRESTORE_IMG):$(TAG) --file - --progress=plain
+
+.PHONY: docker-build-and-push-etcdrestore
+docker-build-and-push-etcdrestore: buildx-machine docker-pull-prerequisites ## Run docker-build-and-push-etcdrestore targets for all architectures
+	cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build \
+			--platform $(TARGET_PLATFORMS) \
+			--push \
+			--sbom=true \
+			--attest type=provenance,mode=max \
+			--iidfile=$(IID_FILE) \
+			--build-arg builder_image=$(GO_CONTAINER_IMAGE) \
+			--build-arg goproxy=$(GOPROXY) \
+			--build-arg package=./exp/etcdrestore \
+			--build-arg ldflags="$(LDFLAGS)" . -t $(ETCDRESTORE_IMG):$(TAG) --file - --progress=plain
+
+.PHONY: docker-build
 docker-build: buildx-machine docker-pull-prerequisites ## Build docker image for a specific architecture
 	# buildx does not support using local registry for multi-architecture images
 	DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build \

PROJECT

@@ -23,7 +23,7 @@ resources:
   controller: true
   domain: cattle.io
   group: turtles-capi
-  kind: EtcdMachineSnapshot
+  kind: ETCDMachineSnapshot
   path: github.com/rancher/turtles/exp/etcdrestore/api/v1alpha1
   version: v1alpha1
 - api:
@@ -32,7 +32,7 @@ resources:
   controller: true
   domain: cattle.io
   group: turtles-capi
-  kind: EtcdSnapshotRestore
+  kind: ETCDSnapshotRestore
   path: github.com/rancher/turtles/exp/etcdrestore/api/v1alpha1
   version: v1alpha1
 - api:

exp/README.md

@@ -12,15 +12,19 @@ To set up the environment, navigate to the root of the repository and run:
 export RANCHER_HOSTNAME="<hostname>"
 export NGROK_API_KEY="<api-key>"
 export NGROK_AUTHTOKEN="<api-authtoken>"
+export USE_TILT_DEV=true (default)
 
 make dev-env
 ```
 
-The `Makefile` target sets up the environment by executing the `scripts/etcd-backup-restore-dev.sh`
+**Note:** setting `USE_TILT_DEV` environment variable to `false` will result in manually deploying Rancher Turtles locally instead
+of Tilt deployment and can be used for testing Rancher Turtles with Helm chart changes (enabling/disabling feature flags when passed as argument to Turtles helm installation command).
+
+The `Makefile` target sets up the environment by executing the `scripts/turtles-dev.sh`
 script with the `RANCHER_HOSTNAME` argument. Under the hood, it performs the following steps:
 
 1. Creates a kind cluster.
-2. Deploys cert-manager, CAPI Operator with Rancher Turtles.
+2. Deploys cert-manager, CAPI Operator and Rancher Turtles.
 3. Deploys CAPRKE2 provider.
 4. Deploys Docker provider.
 5. Deploys ngrok.

exp/etcdrestore/Dockerfile

@@ -0,0 +1,70 @@
+# syntax=docker/dockerfile:1.4
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the etcdrestore binary
+# Run this with docker build --build-arg builder_image=<golang:x.y.z>
+ARG builder_image
+
+# Build architecture
+ARG ARCH
+
+# Ignore Hadolint rule "Always tag the version of an image explicitly."
+# It's an invalid finding since the image is explicitly set in the Makefile.
+# https://github.com/hadolint/hadolint/wiki/DL3006
+# hadolint ignore=DL3006
+FROM ${builder_image} as builder
+
+WORKDIR /workspace
+
+# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
+ARG goproxy=https://proxy.golang.org
+# Run this with docker build --build-arg package=./exp/etcdrestore
+ENV GOPROXY=$goproxy
+
+# Copy the Go Modules manifests
+COPY exp/etcdrestore/go.mod /go.mod
+COPY exp/etcdrestore/go.sum /go.sum
+
+# Cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+
+# Copy the sources
+COPY ./ ./
+
+# Cache the go build into the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+     sh -c "cd exp/etcdrestore && go build ."
+
+# # Build
+ARG ARCH
+ARG ldflags
+
+# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
+    sh -c "cd exp/etcdrestore && ls && go build -trimpath -ldflags \"${ldflags} -extldflags '-static'\" -o manager ${package}"
+
+# Production image
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /
+COPY --from=builder /workspace/exp/etcdrestore/manager .
+# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
+USER 65532
+ENTRYPOINT ["/manager"]

scripts/turtles-dev.sh

@@ -20,7 +20,10 @@ if [ -z "$RANCHER_HOSTNAME" ]; then
 	exit 1
 fi
 
-RANCHER_VERSION=${RANCHER_VERSION:-v2.9.0}
+RANCHER_VERSION=${RANCHER_VERSION:-v2.9.1}
+CLUSTER_NAME=${CLUSTER_NAME:-capi-test}
+ETCD_CONTROLLER_IMAGE=${ETCD_CONTROLLER_IMAGE:-ghcr.io/rancher/turtles-etcd-snapshot-restore}
+ETCD_CONTROLLER_IMAGE_TAG=${ETCD_CONTROLLER_IMAGE_TAG:-dev}
 
 BASEDIR=$(dirname "$0")
 
@@ -77,4 +80,32 @@ kubectl rollout status deployment rancher -n cattle-system --timeout=180s
 kubectl apply -f test/e2e/data/rancher/rancher-service-patch.yaml
 envsubst < test/e2e/data/rancher/rancher-setting-patch.yaml | kubectl apply -f -
 
-tilt up
+# Install the locally build chart of Rancher Turtles
+install_local_rancher_turtles_chart() {
+	# Remove the previous chart directory
+	rm -rf out
+	# Build the chart locally
+	make build-chart
+	# Build the etcdrestore controller image
+	make docker-build-etcdrestore
+	# Load the etcdrestore controller image into the kind cluster
+	kind load docker-image $ETCD_CONTROLLER_IMAGE:$ETCD_CONTROLLER_IMAGE_TAG --name $CLUSTER_NAME
+	# Install the Rancher Turtles using a local chart with 'etcd-snapshot-restore' feature flag enabled
+	# to run etcdrestore controller
+	helm install rancher-turtles out/charts/rancher-turtles \
+		-n rancher-turtles-system \
+		--set cluster-api-operator.enabled=false \
+		--set cluster-api-operator.cluster-api.enabled=false \
+		--set rancherTurtles.features.etcd-snapshot-restore.enabled=true \
+		--dependency-update \
+		--create-namespace --wait \
+		--timeout 180s
+}
+
+if [ "$USE_TILT_DEV" == "true" ]; then
+	echo "Using Tilt for development..."
+	tilt up
+else
+	echo "Installing local Rancher Turtles chart for development..."
+	install_local_rancher_turtles_chart
+fi