Skip to content

Commit

Permalink
Merge pull request #8072 from sbueringer/pr-clusterctl-upgrade-1.0
Browse files Browse the repository at this point in the history
🌱 test/e2e: Add v1.0=>main clusterctl upgrade test
  • Loading branch information
k8s-ci-robot authored Feb 12, 2023
2 parents bc98c72 + 187e416 commit eae0775
Show file tree
Hide file tree
Showing 11 changed files with 259 additions and 8 deletions.
6 changes: 1 addition & 5 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,7 @@ hack/tools/bin
*.test

# E2E test templates
test/e2e/data/infrastructure-docker/main/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v0.3/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v0.4/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v1.2/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v1.3/cluster-template*.yaml
test/e2e/data/infrastructure-docker/**/cluster-template*.yaml

# E2e test extension deployment
test/e2e/data/test-extension/deployment.yaml
Expand Down
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -472,7 +472,7 @@ generate-modules: ## Run go mod tidy to ensure modules are up to date
cd $(TEST_DIR); go mod tidy

.PHONY: generate-e2e-templates
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.3 v0.4 v1.2 v1.3 main) ## Generate cluster templates for all versions
generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.3 v0.4 v1.0 v1.2 v1.3 main) ## Generate cluster templates for all versions

DOCKER_TEMPLATES := test/e2e/data/infrastructure-docker

Expand All @@ -484,6 +484,10 @@ generate-e2e-templates-v0.3: $(KUSTOMIZE)
generate-e2e-templates-v0.4: $(KUSTOMIZE)
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v0.4/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v0.4/cluster-template.yaml

.PHONY: generate-e2e-templates-v1.0
generate-e2e-templates-v1.0: $(KUSTOMIZE)
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.0/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.0/cluster-template.yaml

.PHONY: generate-e2e-templates-v1.2
generate-e2e-templates-v1.2: $(KUSTOMIZE)
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.2/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.2/cluster-template.yaml
Expand Down
2 changes: 1 addition & 1 deletion docs/release/release-tasks.md
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ This comes down to changing occurrences of the old version to the new version, e
1. Goal is that we have clusterctl upgrade tests for the latest stable versions of each contract / for each supported branch. For `v1.4` this means:
* v1alpha3: `v0.3`
* v1alpha4: `v0.4`
* v1beta1: `v1.2`, `v1.3` (will change with each new release)
* v1beta1: `v1.0`, `v1.2`, `v1.3` (will change with each new release)
2. Update providers in `docker.yaml`:
1. Add a new `v1.3.0` entry.
2. Remove providers that are not used anymore (for `v1.4` we don't have to remove any).
Expand Down
24 changes: 24 additions & 0 deletions test/e2e/clusterctl_upgrade_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,30 @@ var _ = Describe("When testing clusterctl upgrades (v0.4=>current)", func() {
})
})

var _ = Describe("When testing clusterctl upgrades (v1.0=>current)", func() {
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
return ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.5/clusterctl-{OS}-{ARCH}",
// We have to pin the providers because with `InitWithProvidersContract` the test would
// use the latest version for the contract (which is v1.3.0 for v1beta1).
InitWithCoreProvider: "cluster-api:v1.0.5",
InitWithBootstrapProviders: []string{"kubeadm:v1.0.5"},
InitWithControlPlaneProviders: []string{"kubeadm:v1.0.5"},
InitWithInfrastructureProviders: []string{"docker:v1.0.5"},
// We have to set this to an empty array as clusterctl v1.0 doesn't support
// runtime extension providers. If we don't do this the test will automatically
// try to deploy the latest version of our test-extension from docker.yaml.
InitWithRuntimeExtensionProviders: []string{},
InitWithKubernetesVersion: "v1.23.13",
}
})
})

var _ = Describe("When testing clusterctl upgrades (v1.2=>current)", func() {
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
return ClusterctlUpgradeSpecInput{
Expand Down
37 changes: 37 additions & 0 deletions test/e2e/config/docker.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,15 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v0.4/metadata.yaml"
- name: v1.0.5 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.5/core-components.yaml"
type: "url"
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.0/metadata.yaml"
- name: v1.2.8 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.8/core-components.yaml"
type: "url"
Expand Down Expand Up @@ -96,6 +105,15 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v0.4/metadata.yaml"
- name: v1.0.5 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.5/bootstrap-components.yaml"
type: "url"
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.0/metadata.yaml"
- name: v1.2.8 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.8/bootstrap-components.yaml"
type: "url"
Expand Down Expand Up @@ -143,6 +161,15 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v0.4/metadata.yaml"
- name: v1.0.5 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.5/control-plane-components.yaml"
type: "url"
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.0/metadata.yaml"
- name: v1.2.8 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.8/control-plane-components.yaml"
type: "url"
Expand Down Expand Up @@ -192,6 +219,16 @@ providers:
files:
- sourcePath: "../data/shared/v0.4/metadata.yaml"
- sourcePath: "../data/infrastructure-docker/v0.4/cluster-template.yaml"
- name: v1.0.5 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.5/infrastructure-components-development.yaml"
type: "url"
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1.0/metadata.yaml"
- sourcePath: "../data/infrastructure-docker/v1.0/cluster-template.yaml"
- name: v1.2.8 # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.8/infrastructure-components-development.yaml"
type: "url"
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/data/infrastructure-docker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ to the version from the current branch (and check that the workload cluster stil
As of today we have clusterctl upgrade tests for the latest stable versions of each contract / for each supported branch, i.e.:
* v1alpha3: v0.3
* v1alpha4: v0.4
* v1beta1: v1.2, v1.3 (will change with each new release)
* v1beta1: v1.0, v1.2, v1.3 (will change with each new release)

We cannot use the same cluster templates for all Cluster API versions as not each Cluster API version supports
the same API fields. For example `KubeadmControlPlane.spec.rolloutBefore.certificatesExpiryDays` was introduced
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
---
# DockerCluster object referenced by the Cluster object
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerCluster
metadata:
name: '${CLUSTER_NAME}'
spec:
failureDomains:
fd1:
controlPlane: true
fd2:
controlPlane: true
fd3:
controlPlane: true
fd4:
controlPlane: false
fd5:
controlPlane: false
fd6:
controlPlane: false
fd7:
controlPlane: false
fd8:
controlPlane: false
---
# Cluster object with
# - Reference to the KubeadmControlPlane object
# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet.
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
name: '${CLUSTER_NAME}'
labels:
cni: "${CLUSTER_NAME}-crs-0"
spec:
clusterNetwork:
services:
cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
pods:
cidrBlocks: ['${DOCKER_POD_CIDRS}']
serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerCluster
name: '${CLUSTER_NAME}'
controlPlaneRef:
kind: KubeadmControlPlane
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
name: "${CLUSTER_NAME}-control-plane"
---
# DockerMachineTemplate object referenced by the KubeadmControlPlane object
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
metadata:
name: "${CLUSTER_NAME}-control-plane"
spec:
template:
spec:
extraMounts:
- containerPath: "/var/run/docker.sock"
hostPath: "/var/run/docker.sock"
---
# KubeadmControlPlane referenced by the Cluster object with
# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test.
kind: KubeadmControlPlane
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
metadata:
name: "${CLUSTER_NAME}-control-plane"
labels:
kcp-adoption.step2: ""
spec:
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
machineTemplate:
infrastructureRef:
kind: DockerMachineTemplate
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
name: "${CLUSTER_NAME}-control-plane"
kubeadmConfigSpec:
clusterConfiguration:
controllerManager:
extraArgs: {enable-hostpath-provisioner: 'true'}
apiServer:
# host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
initConfiguration:
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
kubeletExtraArgs:
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
joinConfiguration:
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
kubeletExtraArgs:
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
version: "${KUBERNETES_VERSION}"
24 changes: 24 additions & 0 deletions test/e2e/data/infrastructure-docker/v1.0/bases/crs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
---
# ConfigMap object referenced by the ClusterResourceSet object and with
# the CNI resource defined in the test config file
apiVersion: v1
kind: ConfigMap
metadata:
name: "cni-${CLUSTER_NAME}-crs-0"
data: ${CNI_RESOURCES}
binaryData:
---
# ClusterResourceSet object with
# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0
apiVersion: addons.cluster.x-k8s.io/v1beta1
kind: ClusterResourceSet
metadata:
name: "${CLUSTER_NAME}-crs-0"
spec:
strategy: ApplyOnce
clusterSelector:
matchLabels:
cni: "${CLUSTER_NAME}-crs-0"
resources:
- name: "cni-${CLUSTER_NAME}-crs-0"
kind: ConfigMap
52 changes: 52 additions & 0 deletions test/e2e/data/infrastructure-docker/v1.0/bases/md.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
---
# DockerMachineTemplate referenced by the MachineDeployment and with
# - extraMounts for the docker sock, thus allowing self-hosting test
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
template:
spec:
extraMounts:
- containerPath: "/var/run/docker.sock"
hostPath: "/var/run/docker.sock"
---
# KubeadmConfigTemplate referenced by the MachineDeployment
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
template:
spec:
joinConfiguration:
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
kubeletExtraArgs:
eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
---
# MachineDeployment object
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineDeployment
metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
clusterName: "${CLUSTER_NAME}"
replicas: ${WORKER_MACHINE_COUNT}
selector:
matchLabels:
template:
spec:
clusterName: "${CLUSTER_NAME}"
version: "${KUBERNETES_VERSION}"
bootstrap:
configRef:
name: "${CLUSTER_NAME}-md-0"
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
infrastructureRef:
name: "${CLUSTER_NAME}-md-0"
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
failureDomain: fd4
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
bases:
- ../bases/cluster-with-kcp.yaml
- ../bases/md.yaml
- ../bases/crs.yaml
15 changes: 15 additions & 0 deletions test/e2e/data/shared/v1.0/metadata.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
kind: Metadata
releaseSeries:
- major: 1
minor: 0
contract: v1beta1
- major: 0
minor: 4
contract: v1alpha4
- major: 0
minor: 3
contract: v1alpha3
- major: 0
minor: 2
contract: v1alpha2

0 comments on commit eae0775

Please sign in to comment.