From 4acd13895a05edd21dd8a5f79c724671609d4743 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 19 Jan 2024 09:12:38 -0800 Subject: [PATCH] [release-1.12] overcome transient failures + retry when creating test cluster Signed-off-by: Jack Francis --- Makefile | 16 ++++++++-------- hack/create-dev-cluster.sh | 36 +++++++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 535966569f9..9b51e198b32 100644 --- a/Makefile +++ b/Makefile @@ -301,14 +301,14 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create ./hack/create-custom-cloud-provider-config.sh # Deploy CAPI - curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.5/cluster-api-components.yaml | $(ENVSUBST) | $(KUBECTL) apply -f - + timeout --foreground 300 bash -c "until curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.5/cluster-api-components.yaml | $(ENVSUBST) | $(KUBECTL) apply -f -; do sleep 5; done" # Deploy CAAPH - curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/download/v0.1.0-alpha.10/addon-components.yaml | $(ENVSUBST) | $(KUBECTL) apply -f - + timeout --foreground 300 bash -c "until curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/download/v0.1.0-alpha.10/addon-components.yaml | $(ENVSUBST) | $(KUBECTL) apply -f -; do sleep 5; done" # Deploy CAPZ $(KIND) load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=$(KIND_CLUSTER_NAME) - $(KUSTOMIZE) build config/default | $(ENVSUBST) | $(KUBECTL) apply -f - --server-side=true + timeout --foreground 300 bash -c "until $(KUSTOMIZE) build config/default | $(ENVSUBST) | $(KUBECTL) apply -f - --server-side=true; do sleep 5; done" # Wait for CAPI deployments $(KUBECTL) wait --for=condition=Available --timeout=5m -n capi-system deployment -l cluster.x-k8s.io/provider=cluster-api @@ -322,8 +322,8 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create timeout --foreground 300 bash -c "until $(KUBECTL) get clusterresourcesets -A; do sleep 3; done" # install Windows Calico cluster resource set - $(KUBECTL) create configmap calico-windows-addon --from-file="$(ADDONS_DIR)/windows/calico" --dry-run=client -o yaml | kubectl apply -f - - $(KUBECTL) apply -f templates/addons/windows/calico-resource-set.yaml + timeout --foreground 300 bash -c "until $(KUBECTL) create configmap calico-windows-addon --from-file=$(ADDONS_DIR)/windows/calico --dry-run=client -o yaml | kubectl apply -f -; do sleep 5; done" + timeout --foreground 300 bash -c "until $(KUBECTL) apply -f templates/addons/windows/calico-resource-set.yaml; do sleep 5; done" # Wait for CAPZ deployments $(KUBECTL) wait --for=condition=Available --timeout=5m -n capz-system deployment --all @@ -339,11 +339,11 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create create-workload-cluster: $(ENVSUBST) $(KUBECTL) ## Create a workload cluster. # Create workload Cluster. @if [ -f "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" ]; then \ - $(ENVSUBST) < "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" | $(KUBECTL) apply -f -; \ + timeout --foreground 300 bash -c "until $(ENVSUBST) < $(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE) | $(KUBECTL) apply -f -; do sleep 5; done"; \ elif [ -f "$(CLUSTER_TEMPLATE)" ]; then \ - $(ENVSUBST) < "$(CLUSTER_TEMPLATE)" | $(KUBECTL) apply -f -; \ + timeout --foreground 300 bash -c "until $(ENVSUBST) < $(CLUSTER_TEMPLATE) | $(KUBECTL) apply -f -; do sleep 5; done"; \ else \ - curl --retry "$(CURL_RETRIES)" "$(CLUSTER_TEMPLATE)" | "$(ENVSUBST)" | $(KUBECTL) apply -f -; \ + timeout --foreground 300 bash -c "until curl --retry $(CURL_RETRIES) $(CLUSTER_TEMPLATE) | $(ENVSUBST) | $(KUBECTL) apply -f -; do sleep 5; done"; \ fi # Wait for the kubeconfig to become available. diff --git a/hack/create-dev-cluster.sh b/hack/create-dev-cluster.sh index 328c515e90c..d230e836e6d 100755 --- a/hack/create-dev-cluster.sh +++ b/hack/create-dev-cluster.sh @@ -28,6 +28,8 @@ make envsubst export REGISTRY="${REGISTRY:-registry.local/fake}" +export CLUSTER_CREATE_ATTEMPTS="${CLUSTER_CREATE_ATTEMPTS:-3}" + # Cluster settings. export CLUSTER_NAME="${CLUSTER_NAME:-capz-test}" export AZURE_VNET_NAME=${CLUSTER_NAME}-vnet @@ -62,14 +64,26 @@ capz::util::generate_ssh_key echo "================ DOCKER BUILD ===============" PULL_POLICY=IfNotPresent make modules docker-build -echo "================ MAKE CLEAN ===============" -make clean - -echo "================ KIND RESET ===============" -make kind-reset - -echo "================ INSTALL TOOLS ===============" -make install-tools - -echo "================ CREATE CLUSTER ===============" -make create-cluster +setup() { + echo "================ MAKE CLEAN ===============" + make clean + + echo "================ KIND RESET ===============" + make kind-reset + + echo "================ INSTALL TOOLS ===============" + make install-tools +} + +create_cluster() { + echo "================ CREATE CLUSTER ===============" + make create-cluster +} + +retries=$CLUSTER_CREATE_ATTEMPTS +while ((retries > 0)); do + create_cluster && break + setup + sleep 5 + ((retries --)) +done