From 78d26ccba2eaa3d063af8a58c9f6ab896b599f2e Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Mon, 6 Feb 2023 19:42:53 +0100 Subject: [PATCH] test/e2e: Add v1.0=>main clusterctl upgrade test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stefan Büringer buringerst@vmware.com --- .gitignore | 6 +- Makefile | 6 +- docs/release/release-tasks.md | 2 +- test/e2e/clusterctl_upgrade_test.go | 24 +++++ test/e2e/config/docker.yaml | 37 ++++++++ test/e2e/data/infrastructure-docker/README.md | 2 +- .../v1.0/bases/cluster-with-kcp.yaml | 95 +++++++++++++++++++ .../infrastructure-docker/v1.0/bases/crs.yaml | 24 +++++ .../infrastructure-docker/v1.0/bases/md.yaml | 52 ++++++++++ .../v1.0/cluster-template/kustomization.yaml | 4 + test/e2e/data/shared/v1.0/metadata.yaml | 15 +++ 11 files changed, 259 insertions(+), 8 deletions(-) create mode 100644 test/e2e/data/infrastructure-docker/v1.0/bases/cluster-with-kcp.yaml create mode 100644 test/e2e/data/infrastructure-docker/v1.0/bases/crs.yaml create mode 100644 test/e2e/data/infrastructure-docker/v1.0/bases/md.yaml create mode 100644 test/e2e/data/infrastructure-docker/v1.0/cluster-template/kustomization.yaml create mode 100644 test/e2e/data/shared/v1.0/metadata.yaml diff --git a/.gitignore b/.gitignore index f39502bcb9b1..6ea28f085240 100644 --- a/.gitignore +++ b/.gitignore @@ -11,11 +11,7 @@ hack/tools/bin *.test # E2E test templates -test/e2e/data/infrastructure-docker/main/cluster-template*.yaml -test/e2e/data/infrastructure-docker/v0.3/cluster-template*.yaml -test/e2e/data/infrastructure-docker/v0.4/cluster-template*.yaml -test/e2e/data/infrastructure-docker/v1.2/cluster-template*.yaml -test/e2e/data/infrastructure-docker/v1.3/cluster-template*.yaml +test/e2e/data/infrastructure-docker/**/cluster-template*.yaml # E2e test extension deployment test/e2e/data/test-extension/deployment.yaml diff --git a/Makefile b/Makefile index cc29adce47b7..c50e87fcf94a 100644 --- a/Makefile +++ b/Makefile @@ -472,7 +472,7 @@ generate-modules: ## Run go mod tidy to ensure modules are up to date cd $(TEST_DIR); go mod tidy .PHONY: generate-e2e-templates -generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.3 v0.4 v1.2 v1.3 main) ## Generate cluster templates for all versions +generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.3 v0.4 v1.0 v1.2 v1.3 main) ## Generate cluster templates for all versions DOCKER_TEMPLATES := test/e2e/data/infrastructure-docker @@ -484,6 +484,10 @@ generate-e2e-templates-v0.3: $(KUSTOMIZE) generate-e2e-templates-v0.4: $(KUSTOMIZE) $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v0.4/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v0.4/cluster-template.yaml +.PHONY: generate-e2e-templates-v1.0 +generate-e2e-templates-v1.0: $(KUSTOMIZE) + $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.0/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v0.4/cluster-template.yaml + .PHONY: generate-e2e-templates-v1.2 generate-e2e-templates-v1.2: $(KUSTOMIZE) $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.2/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.2/cluster-template.yaml diff --git a/docs/release/release-tasks.md b/docs/release/release-tasks.md index 0cdd7efbef12..32d8a7eef81d 100644 --- a/docs/release/release-tasks.md +++ b/docs/release/release-tasks.md @@ -95,7 +95,7 @@ This comes down to changing occurrences of the old version to the new version, e 1. Goal is that we have clusterctl upgrade tests for the latest stable versions of each contract / for each supported branch. For `v1.4` this means: * v1alpha3: `v0.3` * v1alpha4: `v0.4` - * v1beta1: `v1.2`, `v1.3` (will change with each new release) + * v1beta1: `v1.0`, `v1.2`, `v1.3` (will change with each new release) 2. Update providers in `docker.yaml`: 1. Add a new `v1.3.0` entry. 2. Remove providers that are not used anymore (for `v1.4` we don't have to remove any). diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index 805b8f5a1d61..a6aa45cdf7fa 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -58,6 +58,30 @@ var _ = Describe("When testing clusterctl upgrades (v0.4=>current)", func() { }) }) +var _ = Describe("When testing clusterctl upgrades (v1.0=>current)", func() { + ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput { + return ClusterctlUpgradeSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.5/clusterctl-{OS}-{ARCH}", + // We have to pin the providers because with `InitWithProvidersContract` the test would + // use the latest version for the contract (which is v1.3.0 for v1beta1). + InitWithCoreProvider: "cluster-api:v1.0.5", + InitWithBootstrapProviders: []string{"kubeadm:v1.0.5"}, + InitWithControlPlaneProviders: []string{"kubeadm:v1.0.5"}, + InitWithInfrastructureProviders: []string{"docker:v1.0.5"}, + // We have to set this to an empty array as clusterctl v1.2 doesn't support + // runtime extension providers. If we don't do this the test will automatically + // try to deploy the latest version of our test-extension from docker.yaml. + InitWithRuntimeExtensionProviders: []string{}, + InitWithKubernetesVersion: "v1.23.13", + } + }) +}) + var _ = Describe("When testing clusterctl upgrades (v1.2=>current)", func() { ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput { return ClusterctlUpgradeSpecInput{ diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index 708ecef3b2e0..15227d138ead 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -49,6 +49,15 @@ providers: new: --metrics-addr=:8080 files: - sourcePath: "../data/shared/v0.4/metadata.yaml" + - name: v1.0.5 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.5/core-components.yaml" + type: "url" + contract: v1beta1 + replacements: + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + files: + - sourcePath: "../data/shared/v1.0/metadata.yaml" - name: v1.2.8 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.8/core-components.yaml" type: "url" @@ -96,6 +105,15 @@ providers: new: --metrics-addr=:8080 files: - sourcePath: "../data/shared/v0.4/metadata.yaml" + - name: v1.0.5 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.5/bootstrap-components.yaml" + type: "url" + contract: v1beta1 + replacements: + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + files: + - sourcePath: "../data/shared/v1.0/metadata.yaml" - name: v1.2.8 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.8/bootstrap-components.yaml" type: "url" @@ -143,6 +161,15 @@ providers: new: --metrics-addr=:8080 files: - sourcePath: "../data/shared/v0.4/metadata.yaml" + - name: v1.0.5 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.5/control-plane-components.yaml" + type: "url" + contract: v1beta1 + replacements: + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + files: + - sourcePath: "../data/shared/v1.0/metadata.yaml" - name: v1.2.8 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.8/control-plane-components.yaml" type: "url" @@ -192,6 +219,16 @@ providers: files: - sourcePath: "../data/shared/v0.4/metadata.yaml" - sourcePath: "../data/infrastructure-docker/v0.4/cluster-template.yaml" + - name: v1.0.5 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.5/infrastructure-components-development.yaml" + type: "url" + contract: v1beta1 + replacements: + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + files: + - sourcePath: "../data/shared/v1.0/metadata.yaml" + - sourcePath: "../data/infrastructure-docker/v1.0/cluster-template.yaml" - name: v1.2.8 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.8/infrastructure-components-development.yaml" type: "url" diff --git a/test/e2e/data/infrastructure-docker/README.md b/test/e2e/data/infrastructure-docker/README.md index 1c14e0a278bb..3ac5d04ec546 100644 --- a/test/e2e/data/infrastructure-docker/README.md +++ b/test/e2e/data/infrastructure-docker/README.md @@ -10,7 +10,7 @@ to the version from the current branch (and check that the workload cluster stil As of today we have clusterctl upgrade tests for the latest stable versions of each contract / for each supported branch, i.e.: * v1alpha3: v0.3 * v1alpha4: v0.4 -* v1beta1: v1.2, v1.3 (will change with each new release) +* v1beta1: v1.0, v1.2, v1.3 (will change with each new release) We cannot use the same cluster templates for all Cluster API versions as not each Cluster API version supports the same API fields. For example `KubeadmControlPlane.spec.rolloutBefore.certificatesExpiryDays` was introduced diff --git a/test/e2e/data/infrastructure-docker/v1.0/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-docker/v1.0/bases/cluster-with-kcp.yaml new file mode 100644 index 000000000000..fa8720f9833f --- /dev/null +++ b/test/e2e/data/infrastructure-docker/v1.0/bases/cluster-with-kcp.yaml @@ -0,0 +1,95 @@ +--- +# DockerCluster object referenced by the Cluster object +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: '${CLUSTER_NAME}' +spec: + failureDomains: + fd1: + controlPlane: true + fd2: + controlPlane: true + fd3: + controlPlane: true + fd4: + controlPlane: false + fd5: + controlPlane: false + fd6: + controlPlane: false + fd7: + controlPlane: false + fd8: + controlPlane: false +--- +# Cluster object with +# - Reference to the KubeadmControlPlane object +# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet. +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: '${CLUSTER_NAME}' + labels: + cni: "${CLUSTER_NAME}-crs-0" +spec: + clusterNetwork: + services: + cidrBlocks: ['${DOCKER_SERVICE_CIDRS}'] + pods: + cidrBlocks: ['${DOCKER_POD_CIDRS}'] + serviceDomain: '${DOCKER_SERVICE_DOMAIN}' + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: '${CLUSTER_NAME}' + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +# DockerMachineTemplate object referenced by the KubeadmControlPlane object +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" +--- +# KubeadmControlPlane referenced by the Cluster object with +# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test. +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" + labels: + kcp-adoption.step2: "" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: DockerMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + clusterConfiguration: + controllerManager: + extraArgs: {enable-hostpath-provisioner: 'true'} + apiServer: + # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied. + certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal] + initConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + version: "${KUBERNETES_VERSION}" diff --git a/test/e2e/data/infrastructure-docker/v1.0/bases/crs.yaml b/test/e2e/data/infrastructure-docker/v1.0/bases/crs.yaml new file mode 100644 index 000000000000..b1b61237dc62 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/v1.0/bases/crs.yaml @@ -0,0 +1,24 @@ +--- +# ConfigMap object referenced by the ClusterResourceSet object and with +# the CNI resource defined in the test config file +apiVersion: v1 +kind: ConfigMap +metadata: + name: "cni-${CLUSTER_NAME}-crs-0" +data: ${CNI_RESOURCES} +binaryData: +--- +# ClusterResourceSet object with +# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0 +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: "${CLUSTER_NAME}-crs-0" +spec: + strategy: ApplyOnce + clusterSelector: + matchLabels: + cni: "${CLUSTER_NAME}-crs-0" + resources: + - name: "cni-${CLUSTER_NAME}-crs-0" + kind: ConfigMap diff --git a/test/e2e/data/infrastructure-docker/v1.0/bases/md.yaml b/test/e2e/data/infrastructure-docker/v1.0/bases/md.yaml new file mode 100644 index 000000000000..11abbc29248a --- /dev/null +++ b/test/e2e/data/infrastructure-docker/v1.0/bases/md.yaml @@ -0,0 +1,52 @@ +--- +# DockerMachineTemplate referenced by the MachineDeployment and with +# - extraMounts for the docker sock, thus allowing self-hosting test +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" +--- +# KubeadmConfigTemplate referenced by the MachineDeployment +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' +--- +# MachineDeployment object +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + failureDomain: fd4 diff --git a/test/e2e/data/infrastructure-docker/v1.0/cluster-template/kustomization.yaml b/test/e2e/data/infrastructure-docker/v1.0/cluster-template/kustomization.yaml new file mode 100644 index 000000000000..3c0dd51f989c --- /dev/null +++ b/test/e2e/data/infrastructure-docker/v1.0/cluster-template/kustomization.yaml @@ -0,0 +1,4 @@ +bases: +- ../bases/cluster-with-kcp.yaml +- ../bases/md.yaml +- ../bases/crs.yaml diff --git a/test/e2e/data/shared/v1.0/metadata.yaml b/test/e2e/data/shared/v1.0/metadata.yaml new file mode 100644 index 000000000000..6a77a02751b1 --- /dev/null +++ b/test/e2e/data/shared/v1.0/metadata.yaml @@ -0,0 +1,15 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +kind: Metadata +releaseSeries: +- major: 1 + minor: 0 + contract: v1beta1 +- major: 0 + minor: 4 + contract: v1alpha4 +- major: 0 + minor: 3 + contract: v1alpha3 +- major: 0 + minor: 2 + contract: v1alpha2