Skip to content

Commit

Permalink
test/e2e: Add clusterctl upgrade with ClusterClass test
Browse files Browse the repository at this point in the history 
Signed-off-by: Stefan Büringer [email protected]
  • Loading branch information
@sbueringer
sbueringer committed Sep 20, 2022
1 parent fd11302 commit 081d8d2
Show file tree
Hide file tree
Showing 14 changed files with 700 additions and 46 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ hack/tools/bin
# E2E test templates
test/e2e/data/infrastructure-docker/v1alpha3/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v1alpha4/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v1.2/cluster-template*.yaml
test/e2e/data/infrastructure-docker/v1beta1/cluster-template*.yaml

# E2e test extension deployment
Expand Down
6 changes: 5 additions & 1 deletion test/e2e/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ $(KUSTOMIZE_BIN): $(KUSTOMIZE) ## Build a local copy of kustomize
DOCKER_TEMPLATES := $(REPO_ROOT)/test/e2e/data/infrastructure-docker

.PHONY: cluster-templates
cluster-templates: $(KUSTOMIZE) cluster-templates-v1alpha3 cluster-templates-v1alpha4 cluster-templates-v1beta1 ## Generate cluster templates for all versions
cluster-templates: $(KUSTOMIZE) cluster-templates-v1alpha3 cluster-templates-v1alpha4 cluster-templates-v1.2 cluster-templates-v1beta1 ## Generate cluster templates for all versions

cluster-templates-v1alpha3: $(KUSTOMIZE) ## Generate cluster templates for v1alpha3
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha3/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1alpha3/cluster-template.yaml
Expand All @@ -84,6 +84,10 @@ cluster-templates-v1alpha4: $(KUSTOMIZE) ## Generate cluster templates for v1alp
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-kcp-scale-in.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-ipv6 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1alpha4/cluster-template-ipv6.yaml

cluster-templates-v1.2: $(KUSTOMIZE) ## Generate cluster templates for v1.2
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.2/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.2/cluster-template.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.2/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.2/cluster-template-topology.yaml

cluster-templates-v1beta1: $(KUSTOMIZE) ## Generate cluster templates for v1beta1
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template-md-remediation --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template-md-remediation.yaml
Expand Down
126 changes: 91 additions & 35 deletions test/e2e/clusterctl_upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ import (
"runtime"
"strings"

"github.com/gobuffalo/flect"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand Down Expand Up @@ -68,6 +70,9 @@ type ClusterctlUpgradeSpecInput struct {
// InitWithProvidersContract can be used to override the INIT_WITH_PROVIDERS_CONTRACT e2e config variable with a specific
// provider contract to use to initialise the secondary management cluster, e.g. `v1alpha3`
InitWithProvidersContract string
// InitWithKubernetesVersion can be used to override the INIT_WITH_KUBERNETES_VERSION e2e config variable with a specific
// Kubernetes version to use to create the secondary management cluster, e.g. `v1.25.0`
InitWithKubernetesVersion string
SkipCleanup bool
ControlPlaneWaiters clusterctl.ControlPlaneWaiters
PreInit func(managementClusterProxy framework.ClusterProxy)
Expand Down Expand Up @@ -115,12 +120,15 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
testCancelWatches context.CancelFunc

managementClusterName string
clusterctlBinaryURL string
managementClusterNamespace *corev1.Namespace
managementClusterCancelWatches context.CancelFunc
managementClusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
managementClusterProxy framework.ClusterProxy

initClusterctlBinaryURL string
initContract string
initKubernetesVersion string

workLoadClusterName string
)

Expand All @@ -130,6 +138,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)

var clusterctlBinaryURLTemplate string
if input.InitWithBinary == "" {
Expect(input.E2EConfig.Variables).To(HaveKey(initWithBinaryVariableName), "Invalid argument. %s variable must be defined when calling %s spec", initWithBinaryVariableName, specName)
Expand All @@ -139,8 +148,27 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
clusterctlBinaryURLTemplate = input.InitWithBinary
}
clusterctlBinaryURLReplacer := strings.NewReplacer("{OS}", runtime.GOOS, "{ARCH}", runtime.GOARCH)
clusterctlBinaryURL = clusterctlBinaryURLReplacer.Replace(clusterctlBinaryURLTemplate)
Expect(input.E2EConfig.Variables).To(HaveKey(initWithKubernetesVersion))
initClusterctlBinaryURL = clusterctlBinaryURLReplacer.Replace(clusterctlBinaryURLTemplate)

// NOTE: by default we are considering all the providers, no matter of the contract.
// However, given that we want to test both v1alpha3 --> v1beta1 and v1alpha4 --> v1beta1, the INIT_WITH_PROVIDERS_CONTRACT
// variable can be used to select versions with a specific contract.
initContract = "*"
if input.E2EConfig.HasVariable(initWithProvidersContract) {
initContract = input.E2EConfig.GetVariable(initWithProvidersContract)
}
if input.InitWithProvidersContract != "" {
initContract = input.InitWithProvidersContract
}

if input.InitWithKubernetesVersion == "" {
Expect(input.E2EConfig.Variables).To(HaveKey(initWithKubernetesVersion), "Invalid argument. %s variable must be defined when calling %s spec", initWithKubernetesVersion, specName)
Expect(input.E2EConfig.Variables[initWithKubernetesVersion]).ToNot(BeEmpty(), "Invalid argument. %s variable can't be empty when calling %s spec", initWithKubernetesVersion, specName)
initKubernetesVersion = input.E2EConfig.GetVariable(initWithKubernetesVersion)
} else {
initKubernetesVersion = input.InitWithKubernetesVersion
}

Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)

Expand All @@ -164,7 +192,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
Flavor: input.MgmtFlavor,
Namespace: managementClusterNamespace.Name,
ClusterName: managementClusterName,
KubernetesVersion: input.E2EConfig.GetVariable(initWithKubernetesVersion),
KubernetesVersion: initKubernetesVersion,
ControlPlaneMachineCount: pointer.Int64Ptr(1),
WorkerMachineCount: pointer.Int64Ptr(1),
},
Expand Down Expand Up @@ -193,27 +221,15 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
managementClusterProxy = input.BootstrapClusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name)

// Download the older clusterctl version to be used for setting up the management cluster to be upgraded

log.Logf("Downloading clusterctl binary from %s", clusterctlBinaryURL)
clusterctlBinaryPath := downloadToTmpFile(ctx, clusterctlBinaryURL)
log.Logf("Downloading clusterctl binary from %s", initClusterctlBinaryURL)
clusterctlBinaryPath := downloadToTmpFile(ctx, initClusterctlBinaryURL)
defer os.Remove(clusterctlBinaryPath) // clean up

err := os.Chmod(clusterctlBinaryPath, 0744) //nolint:gosec
Expect(err).ToNot(HaveOccurred(), "failed to chmod temporary file")

By("Initializing the workload cluster with older versions of providers")

// NOTE: by default we are considering all the providers, no matter of the contract.
// However, given that we want to test both v1alpha3 --> v1beta1 and v1alpha4 --> v1beta1, the INIT_WITH_PROVIDERS_CONTRACT
// variable can be used to select versions with a specific contract.
contract := "*"
if input.E2EConfig.HasVariable(initWithProvidersContract) {
contract = input.E2EConfig.GetVariable(initWithProvidersContract)
}
if input.InitWithProvidersContract != "" {
contract = input.InitWithProvidersContract
}

if input.PreInit != nil {
By("Running Pre-init steps against the management cluster")
input.PreInit(managementClusterProxy)
Expand All @@ -223,10 +239,10 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
ClusterctlBinaryPath: clusterctlBinaryPath, // use older version of clusterctl to init the management cluster
ClusterProxy: managementClusterProxy,
ClusterctlConfigPath: input.ClusterctlConfigPath,
CoreProvider: input.E2EConfig.GetProviderLatestVersionsByContract(contract, config.ClusterAPIProviderName)[0],
BootstrapProviders: input.E2EConfig.GetProviderLatestVersionsByContract(contract, config.KubeadmBootstrapProviderName),
ControlPlaneProviders: input.E2EConfig.GetProviderLatestVersionsByContract(contract, config.KubeadmControlPlaneProviderName),
InfrastructureProviders: input.E2EConfig.GetProviderLatestVersionsByContract(contract, input.E2EConfig.InfrastructureProviders()...),
CoreProvider: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, config.ClusterAPIProviderName)[0],
BootstrapProviders: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, config.KubeadmBootstrapProviderName),
ControlPlaneProviders: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, config.KubeadmControlPlaneProviderName),
InfrastructureProviders: input.E2EConfig.GetProviderLatestVersionsByContract(initContract, input.E2EConfig.InfrastructureProviders()...),
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name),
}, input.E2EConfig.GetIntervals(specName, "wait-controllers")...)

Expand Down Expand Up @@ -342,19 +358,30 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
// After upgrading we are sure the version is the latest version of the API,
// so it is possible to use the standard helpers

testMachineDeployments := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{
Lister: managementClusterProxy.GetClient(),
ClusterName: workLoadClusterName,
Namespace: testNamespace.Name,
})

framework.ScaleAndWaitMachineDeployment(ctx, framework.ScaleAndWaitMachineDeploymentInput{
ClusterProxy: managementClusterProxy,
Cluster: &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace.Name}},
MachineDeployment: testMachineDeployments[0],
Replicas: 2,
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
})
isTopologyManaged, workloadCluster := clusterIsTopologyManaged(ctx, managementClusterProxy.GetClient(), testNamespace.Name, workLoadClusterName)
if isTopologyManaged {
// Cluster is using ClusterClass, scale up via topology.
framework.ScaleAndWaitMachineDeploymentTopology(ctx, framework.ScaleAndWaitMachineDeploymentTopologyInput{
ClusterProxy: managementClusterProxy,
Cluster: workloadCluster,
Replicas: 2,
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
})
} else {
// Cluster is not using ClusterClass, scale up via MachineDeployment.
testMachineDeployments := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{
Lister: managementClusterProxy.GetClient(),
ClusterName: workLoadClusterName,
Namespace: testNamespace.Name,
})
framework.ScaleAndWaitMachineDeployment(ctx, framework.ScaleAndWaitMachineDeploymentInput{
ClusterProxy: managementClusterProxy,
Cluster: &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace.Name}},
MachineDeployment: testMachineDeployments[0],
Replicas: 2,
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
})
}

By("THE UPGRADED MANAGEMENT CLUSTER WORKS!")

Expand Down Expand Up @@ -443,6 +470,35 @@ func downloadToTmpFile(ctx context.Context, url string) string {
return tmpFile.Name()
}

// clusterIsTopologyManaged determines if the Cluster is topology managed and returns the
// Cluster only if it is.
func clusterIsTopologyManaged(ctx context.Context, managementClusterProxy framework.Getter, clusterNamespace, clusterName string) (bool, *clusterv1.Cluster) {
clusterCRD := apiextensionsv1.CustomResourceDefinition{}
clusterCRDName := fmt.Sprintf("%s.%s", flect.Pluralize(strings.ToLower("Cluster")), clusterv1.GroupVersion.Group)
err := managementClusterProxy.Get(ctx, client.ObjectKey{Name: clusterCRDName}, &clusterCRD)
Expect(err).ToNot(HaveOccurred())

foundV1beta1 := false
for _, version := range clusterCRD.Spec.Versions {
if version.Name == "v1beta1" {
foundV1beta1 = true
break
}
}

// Only the v1beta1 version supports ClusterClass.
if !foundV1beta1 {
return false, nil
}

cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{
Getter: managementClusterProxy,
Namespace: clusterNamespace,
Name: clusterName,
})
return true, cluster
}

// deleteAllClustersAndWaitV1alpha3 deletes all cluster resources in the given namespace and waits for them to be gone using the older API.
func deleteAllClustersAndWaitV1alpha3(ctx context.Context, input framework.DeleteAllClustersAndWaitInput, intervals ...interface{}) {
Expect(ctx).NotTo(BeNil(), "ctx is required for deleteAllClustersAndWaitOldAPI")
Expand Down
16 changes: 16 additions & 0 deletions test/e2e/clusterctl_upgrade_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,19 @@ var _ = Describe("When testing clusterctl upgrades [clusterctl-Upgrade]", func()
}
})
})

var _ = Describe("When testing clusterctl upgrades using ClusterClass", func() {
ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
return ClusterctlUpgradeSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/clusterctl-{OS}-{ARCH}",
InitWithProvidersContract: "v1beta1",
InitWithKubernetesVersion: "v1.25.0",
WorkloadFlavor: "topology",
}
})
})
20 changes: 11 additions & 9 deletions test/e2e/config/docker.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.2.0 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/core-components.yaml"
- name: v1.2.2 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/core-components.yaml"
type: "url"
contract: v1beta1
replacements:
Expand Down Expand Up @@ -87,8 +87,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.2.0 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/bootstrap-components.yaml"
- name: v1.2.2 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/bootstrap-components.yaml"
type: "url"
contract: v1beta1
replacements:
Expand Down Expand Up @@ -125,8 +125,8 @@ providers:
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- name: v1.2.0 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/control-plane-components.yaml"
- name: v1.2.2 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/control-plane-components.yaml"
type: "url"
contract: v1beta1
replacements:
Expand Down Expand Up @@ -165,16 +165,18 @@ providers:
files:
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
- sourcePath: "../data/infrastructure-docker/v1alpha4/cluster-template.yaml"
- name: v1.2.0 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/infrastructure-components-development.yaml"
- name: v1.2.2 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.2/infrastructure-components-development.yaml"
type: "url"
contract: v1beta1
replacements:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
- sourcePath: "../data/shared/v1beta1/metadata.yaml"
- sourcePath: "../data/infrastructure-docker/v1beta1/cluster-template.yaml"
- sourcePath: "../data/infrastructure-docker/v1.2/cluster-template.yaml"
- sourcePath: "../data/infrastructure-docker/v1.2/cluster-template-topology.yaml"
- sourcePath: "../data/infrastructure-docker/v1.2/clusterclass-quick-start.yaml"
- name: v1.3.99 # next; use manifest from source files
value: ../../../test/infrastructure/docker/config/default
replacements:
Expand Down
Loading

0 comments on commit 081d8d2

Please sign in to comment.