🐛 Restore v1alpha3/v1alpha4 conversion to fix SSA issue & add e2e test coverage

Makefile

@@ -562,11 +562,15 @@ generate-doctoc:
 	TRACE=$(TRACE) ./hack/generate-doctoc.sh
 
 .PHONY: generate-e2e-templates
-generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.4 v1.0 v1.5 v1.6 main) ## Generate cluster templates for all versions
+generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.3 v0.4 v1.0 v1.5 v1.6 main) ## Generate cluster templates for all versions
 
 DOCKER_TEMPLATES := test/e2e/data/infrastructure-docker
 INMEMORY_TEMPLATES := test/e2e/data/infrastructure-inmemory
 
+.PHONY: generate-e2e-templates-v0.3
+generate-e2e-templates-v0.3: $(KUSTOMIZE)
+	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v0.3/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v0.3/cluster-template.yaml
+
 .PHONY: generate-e2e-templates-v0.4
 generate-e2e-templates-v0.4: $(KUSTOMIZE)
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v0.4/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v0.4/cluster-template.yaml

test/e2e/clusterctl_upgrade.go

@@ -481,6 +481,9 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
 			Expect(err).ToNot(HaveOccurred())
 
 			clusterctlUpgradeBinaryPath := ""
+			// TODO: While it is generally fine to use this clusterctl config for upgrades as well,
+			// it is not ideal because it points to the latest repositories (e.g. _artifacts/repository/cluster-api/latest/components.yaml)
+			// For example this means if we upgrade to v1.5 the upgrade won't use the metadata.yaml from v1.5 it will use the one from latest.
 			clusterctlUpgradeConfigPath := input.ClusterctlConfigPath
 			if upgrade.WithBinary != "" {
 				// Download the clusterctl version to be used to upgrade the management cluster
@@ -793,8 +796,12 @@ func calculateExpectedWorkerCount(ctx context.Context, c client.Client, unstruct
 	}
 
 	machinePoolList := &unstructured.UnstructuredList{}
+	machinePoolGroup := clusterv1.GroupVersion.Group
+	if coreCAPIStorageVersion == "v1alpha3" {
+		machinePoolGroup = "exp.cluster.x-k8s.io"
+	}
 	machinePoolList.SetGroupVersionKind(schema.GroupVersionKind{
-		Group:   clusterv1.GroupVersion.Group,
+		Group:   machinePoolGroup,
 		Version: coreCAPIStorageVersion,
 		Kind:    "MachinePoolList",
 	})

test/e2e/clusterctl_upgrade_test.go

@@ -21,6 +21,7 @@ package e2e
 
 import (
 	"fmt"
+	"runtime"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -36,6 +37,73 @@ var (
 	providerDockerPrefix  = "docker:v%s"
 )
 
+var _ = Describe("When testing clusterctl upgrades (v0.3=>v1.6=>current)", func() {
+	// Get v0.3 latest stable release
+	version03 := "0.3"
+	stableRelease03, err := GetStableReleaseOfMinor(ctx, version03)
+	Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", version03)
+	clusterctlDownloadURL03 := clusterctlDownloadURL
+	if runtime.GOOS == "darwin" {
+		// There is no arm64 binary for v0.3.x, so we'll use the amd64 one.
+		clusterctlDownloadURL03 = "https://github.com/kubernetes-sigs/cluster-api/releases/download/v%s/clusterctl-darwin-amd64"
+	}
+
+	// Get v1.5 latest stable release
+	version15 := "1.5"
+	stableRelease15, err := GetStableReleaseOfMinor(ctx, version15)
+	Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", version15)
+
+	ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
+		return ClusterctlUpgradeSpecInput{
+			E2EConfig:              e2eConfig,
+			ClusterctlConfigPath:   clusterctlConfigPath,
+			BootstrapClusterProxy:  bootstrapClusterProxy,
+			ArtifactFolder:         artifactFolder,
+			SkipCleanup:            skipCleanup,
+			InfrastructureProvider: ptr.To("docker"),
+			// Configuration for the initial provider deployment.
+			InitWithBinary: fmt.Sprintf(clusterctlDownloadURL03, stableRelease03),
+			// We have to pin the providers because with `InitWithProvidersContract` the test would
+			// use the latest version for the contract.
+			InitWithCoreProvider:            fmt.Sprintf(providerCAPIPrefix, stableRelease03),
+			InitWithBootstrapProviders:      []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease03)},
+			InitWithControlPlaneProviders:   []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease03)},
+			InitWithInfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease03)},
+			// We have to set this to an empty array as clusterctl v0.3 doesn't support
+			// runtime extension providers. If we don't do this the test will automatically
+			// try to deploy the latest version of our test-extension from docker.yaml.
+			InitWithRuntimeExtensionProviders: []string{},
+			// Configuration for the provider upgrades.
+			Upgrades: []ClusterctlUpgradeSpecInputUpgrade{
+				{
+					// Upgrade to v1.5.
+					// Note: v1.5 is the highest version we can use as it's the last one
+					// that is able to upgrade from a v1alpha3 management cluster.
+					WithBinary:              fmt.Sprintf(clusterctlDownloadURL, stableRelease15),
+					CoreProvider:            fmt.Sprintf(providerCAPIPrefix, stableRelease15),
+					BootstrapProviders:      []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease15)},
+					ControlPlaneProviders:   []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease15)},
+					InfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease15)},
+				},
+				{ // Upgrade to latest v1beta1.
+					Contract: clusterv1.GroupVersion.Version,
+				},
+			},
+			// CAPI v0.3.x does not work on Kubernetes >= v1.22.
+			// NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v0.3/bases.
+			//  Note: Both InitWithKubernetesVersion and WorkloadKubernetesVersion should be the highest mgmt cluster version supported by the source Cluster API version.
+			InitWithKubernetesVersion: "v1.21.14",
+			WorkloadKubernetesVersion: "v1.22.17",
+			// CAPI does not work with Kubernetes < v1.22 if ClusterClass is enabled, so we have to disable it.
+			UpgradeClusterctlVariables: map[string]string{
+				"CLUSTER_TOPOLOGY": "false",
+			},
+			MgmtFlavor:     "topology",
+			WorkloadFlavor: "",
+		}
+	})
+})
+
 var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current)", func() {
 	// Get v0.4 latest stable release
 	version04 := "0.4"
@@ -58,7 +126,7 @@ var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current)", func(
 			// Configuration for the initial provider deployment.
 			InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, stableRelease04),
 			// We have to pin the providers because with `InitWithProvidersContract` the test would
-			// use the latest version for the contract (which is v1.3.X for v1beta1).
+			// use the latest version for the contract.
 			InitWithCoreProvider:            fmt.Sprintf(providerCAPIPrefix, stableRelease04),
 			InitWithBootstrapProviders:      []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease04)},
 			InitWithControlPlaneProviders:   []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease04)},
@@ -69,7 +137,10 @@ var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current)", func(
 			InitWithRuntimeExtensionProviders: []string{},
 			// Configuration for the provider upgrades.
 			Upgrades: []ClusterctlUpgradeSpecInputUpgrade{
-				{ // Upgrade to 1.6.
+				{
+					// Upgrade to v1.6.
+					// Note: v1.6 is the highest version we can use as it's the last one
+					// that is able to upgrade from a v1alpha4 management cluster.
 					WithBinary:              fmt.Sprintf(clusterctlDownloadURL, stableRelease16),
 					CoreProvider:            fmt.Sprintf(providerCAPIPrefix, stableRelease16),
 					BootstrapProviders:      []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease16)},

test/e2e/config/docker.yaml

@@ -35,6 +35,15 @@ providers:
 - name: cluster-api
   type: CoreProvider
   versions:
+  - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
+    value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/core-components.yaml"
+    type: "url"
+    contract: v1alpha3
+    replacements:
+    - old: --metrics-addr=127.0.0.1:8080
+      new: --metrics-addr=:8080
+    files:
+    - sourcePath: "../data/shared/v0.3/metadata.yaml"
   - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
     value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/core-components.yaml"
     type: "url"
@@ -82,6 +91,15 @@ providers:
 - name: kubeadm
   type: BootstrapProvider
   versions:
+  - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
+    value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/bootstrap-components.yaml"
+    type: "url"
+    contract: v1alpha3
+    replacements:
+    - old: --metrics-addr=127.0.0.1:8080
+      new: --metrics-addr=:8080
+    files:
+    - sourcePath: "../data/shared/v0.3/metadata.yaml"
   - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
     value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/bootstrap-components.yaml"
     type: "url"
@@ -129,6 +147,15 @@ providers:
 - name: kubeadm
   type: ControlPlaneProvider
   versions:
+  - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
+    value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/control-plane-components.yaml"
+    type: "url"
+    contract: v1alpha3
+    replacements:
+    - old: --metrics-addr=127.0.0.1:8080
+      new: --metrics-addr=:8080
+    files:
+    - sourcePath: "../data/shared/v0.3/metadata.yaml"
   - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
     value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/control-plane-components.yaml"
     type: "url"
@@ -176,6 +203,16 @@ providers:
 - name: docker
   type: InfrastructureProvider
   versions:
+  - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only.
+    value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/infrastructure-components-development.yaml"
+    type: "url"
+    contract: v1alpha3
+    replacements:
+    - old: --metrics-addr=127.0.0.1:8080
+      new: --metrics-addr=:8080
+    files:
+    - sourcePath: "../data/shared/v0.3/metadata.yaml"
+    - sourcePath: "../data/infrastructure-docker/v0.3/cluster-template.yaml"
   - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
     value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/infrastructure-components-development.yaml"
     type: "url"

test/e2e/data/infrastructure-docker/v0.3/bases/cluster-with-kcp.yaml

@@ -0,0 +1,86 @@
+---
+# DockerCluster object referenced by the Cluster object
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: DockerCluster
+metadata:
+  name: '${CLUSTER_NAME}'
+---
+# Cluster object with
+# - Reference to the KubeadmControlPlane object
+# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet.
+apiVersion: cluster.x-k8s.io/v1alpha3
+kind: Cluster
+metadata:
+  name: '${CLUSTER_NAME}'
+  labels:
+    cni: "${CLUSTER_NAME}-crs-0"
+spec:
+  clusterNetwork:
+    services:
+      cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
+    pods:
+      cidrBlocks: ['${DOCKER_POD_CIDRS}']
+    serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+    kind: DockerCluster
+    name: '${CLUSTER_NAME}'
+  controlPlaneRef:
+    kind: KubeadmControlPlane
+    apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
+    name: "${CLUSTER_NAME}-control-plane"
+---
+# DockerMachineTemplate object referenced by the KubeadmControlPlane object
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: DockerMachineTemplate
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+spec:
+  template:
+    spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.22.17@sha256:9af784f45a584f6b28bce2af84c494d947a05bd709151466489008f80a9ce9d5"
+      extraMounts:
+        - containerPath: "/var/run/docker.sock"
+          hostPath: "/var/run/docker.sock"
+---
+# KubeadmControlPlane referenced by the Cluster object with
+# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test.
+kind: KubeadmControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+  labels:
+    kcp-adoption.step2: ""
+spec:
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  infrastructureTemplate:
+    kind: DockerMachineTemplate
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+    name: "${CLUSTER_NAME}-control-plane"
+  kubeadmConfigSpec:
+    clusterConfiguration:
+      controllerManager:
+        extraArgs: {enable-hostpath-provisioner: 'true'}
+      apiServer:
+        # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
+        certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
+    initConfiguration:
+      nodeRegistration:
+        criSocket: unix:///var/run/containerd/containerd.sock
+        kubeletExtraArgs:
+          # We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
+          # This cluster is used in tests where the Kubernetes version is < 1.24
+          cgroup-driver: cgroupfs
+          eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
+          fail-swap-on: "false"
+    joinConfiguration:
+      nodeRegistration:
+        criSocket: unix:///var/run/containerd/containerd.sock
+        kubeletExtraArgs:
+          # We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
+          # This cluster is used in tests where the Kubernetes version is < 1.24
+          cgroup-driver: cgroupfs
+          eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
+          fail-swap-on: "false"
+  version: "${KUBERNETES_VERSION}"

test/e2e/data/infrastructure-docker/v0.3/bases/crs.yaml

@@ -0,0 +1,24 @@
+---
+# ConfigMap object referenced by the ClusterResourceSet object and with
+# the CNI resource defined in the test config file
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "cni-${CLUSTER_NAME}-crs-0"
+data: ${CNI_RESOURCES}
+binaryData:
+---
+# ClusterResourceSet object with
+# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0
+apiVersion: addons.cluster.x-k8s.io/v1alpha3
+kind: ClusterResourceSet
+metadata:
+  name:  "${CLUSTER_NAME}-crs-0"
+spec:
+  strategy: ApplyOnce
+  clusterSelector:
+    matchLabels:
+      cni: "${CLUSTER_NAME}-crs-0"
+  resources:
+    - name: "cni-${CLUSTER_NAME}-crs-0"
+      kind: ConfigMap

test/e2e/data/infrastructure-docker/v0.3/bases/md.yaml

@@ -0,0 +1,57 @@
+---
+# DockerMachineTemplate referenced by the MachineDeployment and with
+# - extraMounts for the docker sock, thus allowing self-hosting test
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: DockerMachineTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec:
+      # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+      customImage: "kindest/node:v1.22.17@sha256:9af784f45a584f6b28bce2af84c494d947a05bd709151466489008f80a9ce9d5"
+      extraMounts:
+        - containerPath: "/var/run/docker.sock"
+          hostPath: "/var/run/docker.sock"
+---
+# KubeadmConfigTemplate referenced by the MachineDeployment
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
+kind: KubeadmConfigTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec:
+      joinConfiguration:
+        nodeRegistration:
+          criSocket: unix:///var/run/containerd/containerd.sock
+          kubeletExtraArgs:
+            eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
+---
+# MachineDeployment object with
+# - the label nodepool=pool1 that applies to all the machines, so those machine can be targeted by the MachineHealthCheck object
+apiVersion: cluster.x-k8s.io/v1alpha3
+kind: MachineDeployment
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+  template:
+    metadata:
+      labels:
+        "nodepool": "pool1"
+    spec:
+      clusterName: "${CLUSTER_NAME}"
+      version: "${KUBERNETES_VERSION}"
+      bootstrap:
+        configRef:
+          name: "${CLUSTER_NAME}-md-0"
+          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
+          kind: KubeadmConfigTemplate
+      infrastructureRef:
+        name: "${CLUSTER_NAME}-md-0"
+        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+        kind: DockerMachineTemplate

test/e2e/data/infrastructure-docker/v0.3/cluster-template/kustomization.yaml

@@ -0,0 +1,4 @@
+bases:
+- ../bases/cluster-with-kcp.yaml
+- ../bases/md.yaml
+- ../bases/crs.yaml

test/e2e/data/shared/main/metadata.yaml

@@ -27,4 +27,7 @@ releaseSeries:
     contract: v1beta1
   - major: 0
     minor: 4
-    contract: v1alpha4
+    contract: v1alpha4
+  - major: 0
+    minor: 3
+    contract: v1alpha3