Merge pull request #4094 from fabriziopandini/KCP-remediation-E2E-test

🌱 Add E2E test for KCP remediation
kubernetes-sigs · Jan 21, 2021 · f92d4ba · f92d4ba
2 parents f7e3629 + e22d361
commit f92d4ba
Show file tree

Hide file tree

Showing 10 changed files with 95 additions and 31 deletions.
diff --git a/test/e2e/Makefile b/test/e2e/Makefile
@@ -60,7 +60,8 @@ cluster-templates-v1alpha3: $(KUSTOMIZE) ## Generate cluster templates for v1alp
 
 cluster-templates-v1alpha4: $(KUSTOMIZE) ## Generate cluster templates for v1alpha4
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template --load_restrictor none > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template.yaml
-	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-mhc --load_restrictor none > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-mhc.yaml
+	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-md-remediation --load_restrictor none > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-md-remediation.yaml
+	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-remediation --load_restrictor none > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-remediation.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-adoption/step1 --load_restrictor none > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-adoption.yaml
 	echo "---" >> $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-adoption.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-adoption/step2 --load_restrictor none >> $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-adoption.yaml

diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml
@@ -74,7 +74,8 @@ providers:
     files:
     # Add cluster templates
     - sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template.yaml"
-    - sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template-mhc.yaml"
+    - sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template-md-remediation.yaml"
+    - sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template-kcp-remediation.yaml"
     - sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template-kcp-adoption.yaml"
     - sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template-machine-pool.yaml"
     - sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template-node-drain.yaml"

diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/bases/md.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/bases/md.yaml
@@ -25,8 +25,7 @@ spec:
           criSocket: /var/run/containerd/containerd.sock
           kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
 ---
-# MachineDeployment object with
-# - the label nodepool=pool1 that applies to all the machines, so those machine can be targeted by the MachineHealthCheck object
+# MachineDeployment object
 apiVersion: cluster.x-k8s.io/v1alpha4
 kind: MachineDeployment
 metadata:
@@ -37,9 +36,6 @@ spec:
   selector:
     matchLabels:
   template:
-    metadata:
-      labels:
-        "nodepool": "pool1"
     spec:
       clusterName: "${CLUSTER_NAME}"
       version: "${KUBERNETES_VERSION}"

diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/bases/mhc.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/bases/mhc.yaml
diff --git a/...4/cluster-template-mhc/kustomization.yaml → ...mplate-kcp-remediation/kustomization.yaml b/...4/cluster-template-mhc/kustomization.yaml → ...mplate-kcp-remediation/kustomization.yaml
@@ -2,6 +2,4 @@ bases:
   - ../bases/cluster-with-kcp.yaml
   - ../bases/md.yaml
   - ../bases/crs.yaml
-  - ../bases/mhc.yaml
-
-
+  - mhc.yaml
diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-kcp-remediation/mhc.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-kcp-remediation/mhc.yaml
@@ -0,0 +1,18 @@
+---
+# MachineHealthCheck object with
+# - a selector that targets all the machines with label cluster.x-k8s.io/control-plane=""
+# - unhealthyConditions triggering remediation after 10s the condition is set
+apiVersion: cluster.x-k8s.io/v1alpha4
+kind: MachineHealthCheck
+metadata:
+  name: "${CLUSTER_NAME}-mhc-0"
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  maxUnhealthy: 100%
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/control-plane: ""
+  unhealthyConditions:
+    - type: e2e.remediation.condition
+      status: "False"
+      timeout: 10s
diff --git a/...2e/data/infrastructure-docker/v1alpha4/cluster-template-md-remediation/kustomization.yaml b/...2e/data/infrastructure-docker/v1alpha4/cluster-template-md-remediation/kustomization.yaml
@@ -0,0 +1,8 @@
+bases:
+  - ../bases/cluster-with-kcp.yaml
+  - ../bases/md.yaml
+  - ../bases/crs.yaml
+  - mhc.yaml
+
+patchesStrategicMerge:
+- ./md.yaml
diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-md-remediation/md.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-md-remediation/md.yaml
@@ -0,0 +1,9 @@
+apiVersion: cluster.x-k8s.io/v1alpha4
+kind: MachineDeployment
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    metadata:
+      labels:
+        "e2e.remediation.label": ""
diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-md-remediation/mhc.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-md-remediation/mhc.yaml
@@ -0,0 +1,18 @@
+---
+# MachineHealthCheck object with
+# - a selector that targets all the machines with label e2e.remediation.label=""
+# - unhealthyConditions triggering remediation after 10s the condition is set
+apiVersion: cluster.x-k8s.io/v1alpha4
+kind: MachineHealthCheck
+metadata:
+  name: "${CLUSTER_NAME}-mhc-0"
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  maxUnhealthy: 100%
+  selector:
+    matchLabels:
+      e2e.remediation.label: ""
+  unhealthyConditions:
+    - type: e2e.remediation.condition
+      status: "False"
+      timeout: 10s
diff --git a/test/e2e/mhc_remediations.go b/test/e2e/mhc_remediations.go
@@ -65,7 +65,7 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed
 		namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
 	})
 
-	It("Should successfully remediate unhealthy machines with MachineHealthCheck", func() {
+	It("Should successfully trigger machine deployment remediation", func() {
 
 		By("Creating a workload cluster")
 
@@ -76,7 +76,7 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed
 				ClusterctlConfigPath:     input.ClusterctlConfigPath,
 				KubeconfigPath:           input.BootstrapClusterProxy.GetKubeconfigPath(),
 				InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
-				Flavor:                   "mhc",
+				Flavor:                   "md-remediation",
 				Namespace:                namespace.Name,
 				ClusterName:              fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
 				KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersion),
@@ -88,7 +88,40 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed
 			WaitForMachineDeployments:    input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
 		})
 
-		By("Waiting for MachineHealthCheck remediation")
+		By("Setting a machine unhealthy and wait for MachineDeployment remediation")
+		framework.DiscoverMachineHealthChecksAndWaitForRemediation(ctx, framework.DiscoverMachineHealthCheckAndWaitForRemediationInput{
+			ClusterProxy:              input.BootstrapClusterProxy,
+			Cluster:                   clusterResources.Cluster,
+			WaitForMachineRemediation: input.E2EConfig.GetIntervals(specName, "wait-machine-remediation"),
+		})
+
+		By("PASSED!")
+	})
+
+	It("Should successfully trigger KCP remediation", func() {
+
+		By("Creating a workload cluster")
+
+		clusterResources = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
+			ClusterProxy: input.BootstrapClusterProxy,
+			ConfigCluster: clusterctl.ConfigClusterInput{
+				LogFolder:                filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
+				ClusterctlConfigPath:     input.ClusterctlConfigPath,
+				KubeconfigPath:           input.BootstrapClusterProxy.GetKubeconfigPath(),
+				InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
+				Flavor:                   "kcp-remediation",
+				Namespace:                namespace.Name,
+				ClusterName:              fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
+				KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersion),
+				ControlPlaneMachineCount: pointer.Int64Ptr(3),
+				WorkerMachineCount:       pointer.Int64Ptr(1),
+			},
+			WaitForClusterIntervals:      input.E2EConfig.GetIntervals(specName, "wait-cluster"),
+			WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
+			WaitForMachineDeployments:    input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
+		})
+
+		By("Setting a machine unhealthy and wait for KubeadmControlPlane remediation")
 		framework.DiscoverMachineHealthChecksAndWaitForRemediation(ctx, framework.DiscoverMachineHealthCheckAndWaitForRemediationInput{
 			ClusterProxy:              input.BootstrapClusterProxy,
 			Cluster:                   clusterResources.Cluster,