Add upgrade tests

Signed-off-by: Stefan Büringer [email protected]
kubernetes-sigs · Jan 3, 2023 · 6c1fa30 · 6c1fa30
1 parent 0587101
commit 6c1fa30
Show file tree

Hide file tree

Showing 5 changed files with 251 additions and 75 deletions.
diff --git a/scripts/ci-e2e-lib.sh b/scripts/ci-e2e-lib.sh
@@ -37,6 +37,19 @@ capi:buildDockerImages () {
 # k8s::prepareKindestImages checks all the e2e test variables representing a Kubernetes version,
 # and makes sure a corresponding kindest/node image is available locally.
 k8s::prepareKindestImages() {
+
+  # Build non-existent images for tests
+  kind::prepareKindestImage "v1.22.16"
+  kind::prepareKindestImage "v1.22.17"
+
+  kind::prepareKindestImage "v1.23.14"
+  kind::prepareKindestImage "v1.23.15"
+
+  kind::prepareKindestImage "v1.24.8"
+  kind::prepareKindestImage "v1.24.9"
+
+
+
   if [ -n "${KUBERNETES_VERSION_MANAGEMENT:-}" ]; then
     k8s::resolveVersion "KUBERNETES_VERSION_MANAGEMENT" "$KUBERNETES_VERSION_MANAGEMENT"
     export KUBERNETES_VERSION_MANAGEMENT=$resolveVersion
@@ -84,6 +97,7 @@ k8s::resolveVersion() {
 
   resolveVersion=$version
   if [[ "$version" =~ ^v ]]; then
+    echo "+ $variableName=\"$version\" resolved to \"$resolveVersion\""
     return
   fi
 
@@ -115,6 +129,15 @@ kind::prepareKindestImage() {
 kind::buildNodeImage() {
   local version=$1
 
+  # Create the Kubernetes repo if it doesn't exist (apparently it doesn't in e2e-main).
+  if [ ! -d "$GOPATH/src/k8s.io/kubernetes" ]; then
+    mkdir -p "$GOPATH/src/k8s.io"
+    cd "$GOPATH/src/k8s.io" || exit
+    git clone https://github.com/kubernetes/kubernetes/
+    # move back to Cluster API
+    cd "$REPO_ROOT" || exit
+  fi
+
   # move to the Kubernetes repository.
   echo "KUBE_ROOT $GOPATH/src/k8s.io/kubernetes"
   cd "$GOPATH/src/k8s.io/kubernetes" || exit

diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh
@@ -53,7 +53,9 @@ kind:prepullAdditionalImages
 # Configure e2e tests
 export GINKGO_NODES=3
 export GINKGO_NOCOLOR=true
-export GINKGO_ARGS="--fail-fast" # Other ginkgo args that need to be appended to the command.
+#export GINKGO_ARGS="--fail-fast" # Other ginkgo args that need to be appended to the command.
+# No fail fast to get results for all tests.
+export GINKGO_ARGS="" # Other ginkgo args that need to be appended to the command.
 export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/docker.yaml"
 export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
 export SKIP_RESOURCE_CLEANUP=false

diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go
@@ -55,6 +55,11 @@ type ClusterUpgradeConformanceSpecInput struct {
 
 	// Flavor to use when creating the cluster for testing, "upgrades" is used if not specified.
 	Flavor *string
+
+	KubernetesVersionUpgradeFrom *string
+	KubernetesVersionUpgradeTo   *string
+	EtcdVersionUpgradeTo         *string
+	CoreDNSVersionUpgradeTo      *string
 }
 
 // ClusterUpgradeConformanceSpec implements a spec that upgrades a cluster and runs the Kubernetes conformance suite.
@@ -118,6 +123,11 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
 	It("Should create and upgrade a workload cluster and eventually run kubetest", func() {
 		By("Creating a workload cluster")
 
+		kubernetesVersionUpgradeFrom := input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom)
+		if input.KubernetesVersionUpgradeFrom != nil {
+			kubernetesVersionUpgradeFrom = *input.KubernetesVersionUpgradeFrom
+		}
+
 		clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
 			ClusterProxy: input.BootstrapClusterProxy,
 			ConfigCluster: clusterctl.ConfigClusterInput{
@@ -128,7 +138,7 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
 				Flavor:                   pointer.StringDeref(input.Flavor, "upgrades"),
 				Namespace:                namespace.Name,
 				ClusterName:              fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
-				KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom),
+				KubernetesVersion:        kubernetesVersionUpgradeFrom,
 				ControlPlaneMachineCount: pointer.Int64(controlPlaneMachineCount),
 				WorkerMachineCount:       pointer.Int64(workerMachineCount),
 			},
@@ -139,17 +149,30 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
 			WaitForMachinePools:          input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
 		}, clusterResources)
 
+		kubernetesVersionUpgradeTo := input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo)
+		if input.KubernetesVersionUpgradeTo != nil {
+			kubernetesVersionUpgradeTo = *input.KubernetesVersionUpgradeTo
+		}
+		etcdVersionUpgradeTo := input.E2EConfig.GetVariable(EtcdVersionUpgradeTo)
+		if input.EtcdVersionUpgradeTo != nil {
+			etcdVersionUpgradeTo = *input.EtcdVersionUpgradeTo
+		}
+		coreDNSVersionUpgradeTo := input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo)
+		if input.CoreDNSVersionUpgradeTo != nil {
+			coreDNSVersionUpgradeTo = *input.CoreDNSVersionUpgradeTo
+		}
+
 		if clusterResources.Cluster.Spec.Topology != nil {
 			// Cluster is using ClusterClass, upgrade via topology.
 			By("Upgrading the Cluster topology")
 			framework.UpgradeClusterTopologyAndWaitForUpgrade(ctx, framework.UpgradeClusterTopologyAndWaitForUpgradeInput{
 				ClusterProxy:                input.BootstrapClusterProxy,
 				Cluster:                     clusterResources.Cluster,
 				ControlPlane:                clusterResources.ControlPlane,
-				EtcdImageTag:                input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
-				DNSImageTag:                 input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
+				EtcdImageTag:                etcdVersionUpgradeTo,
+				DNSImageTag:                 coreDNSVersionUpgradeTo,
 				MachineDeployments:          clusterResources.MachineDeployments,
-				KubernetesUpgradeVersion:    input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
+				KubernetesUpgradeVersion:    kubernetesVersionUpgradeTo,
 				WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
 				WaitForKubeProxyUpgrade:     input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
 				WaitForDNSUpgrade:           input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
@@ -175,9 +198,9 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
 				ClusterProxy:                input.BootstrapClusterProxy,
 				Cluster:                     clusterResources.Cluster,
 				ControlPlane:                clusterResources.ControlPlane,
-				EtcdImageTag:                input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
-				DNSImageTag:                 input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
-				KubernetesUpgradeVersion:    input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
+				EtcdImageTag:                etcdVersionUpgradeTo,
+				DNSImageTag:                 coreDNSVersionUpgradeTo,
+				KubernetesUpgradeVersion:    kubernetesVersionUpgradeTo,
 				UpgradeMachineTemplate:      upgradeCPMachineTemplateTo,
 				WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
 				WaitForKubeProxyUpgrade:     input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
@@ -190,7 +213,7 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
 				framework.UpgradeMachineDeploymentsAndWait(ctx, framework.UpgradeMachineDeploymentsAndWaitInput{
 					ClusterProxy:                input.BootstrapClusterProxy,
 					Cluster:                     clusterResources.Cluster,
-					UpgradeVersion:              input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
+					UpgradeVersion:              kubernetesVersionUpgradeTo,
 					UpgradeMachineTemplate:      upgradeWorkersMachineTemplateTo,
 					MachineDeployments:          clusterResources.MachineDeployments,
 					WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
@@ -204,7 +227,7 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
 			framework.UpgradeMachinePoolAndWait(ctx, framework.UpgradeMachinePoolAndWaitInput{
 				ClusterProxy:                   input.BootstrapClusterProxy,
 				Cluster:                        clusterResources.Cluster,
-				UpgradeVersion:                 input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
+				UpgradeVersion:                 kubernetesVersionUpgradeTo,
 				WaitForMachinePoolToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-upgrade"),
 				MachinePools:                   clusterResources.MachinePools,
 			})
@@ -215,7 +238,7 @@ func ClusterUpgradeConformanceSpec(ctx context.Context, inputGetter func() Clust
 		workloadClient := workloadProxy.GetClient()
 		framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{
 			Lister:            workloadClient,
-			KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
+			KubernetesVersion: kubernetesVersionUpgradeTo,
 			Count:             int(clusterResources.ExpectedTotalNodes()),
 			WaitForNodesReady: input.E2EConfig.GetIntervals(specName, "wait-nodes-ready"),
 		})