From 4bcbca10c475a5e6ef245563aa935941b1cc9c64 Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Fri, 13 Aug 2021 17:37:04 -0400 Subject: [PATCH] [e2e framework] Add ability to run pre and post actions during clusterctl upgrade spec Also enable better cleanup in clusterctl upgrade spec when management cluster hasn't been upgraded yet. --- test/e2e/clusterctl_upgrade.go | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/test/e2e/clusterctl_upgrade.go b/test/e2e/clusterctl_upgrade.go index 572998c2368d..56df09a3310e 100644 --- a/test/e2e/clusterctl_upgrade.go +++ b/test/e2e/clusterctl_upgrade.go @@ -30,7 +30,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" "k8s.io/utils/pointer" clusterv1old "sigs.k8s.io/cluster-api/api/v1alpha3" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" @@ -52,6 +54,8 @@ type ClusterctlUpgradeSpecInput struct { BootstrapClusterProxy framework.ClusterProxy ArtifactFolder string SkipCleanup bool + PreUpgrade func(managementClusterProxy framework.ClusterProxy) + PostUpgrade func(managementClusterProxy framework.ClusterProxy) } // ClusterctlUpgradeSpec implements a test that verifies clusterctl upgrade of a management cluster. @@ -219,6 +223,11 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg By("THE MANAGEMENT CLUSTER WITH OLDER VERSION OF PROVIDERS WORKS!") + if input.PreUpgrade != nil { + By("Running Pre-upgrade steps against the management cluster") + input.PreUpgrade(managementClusterProxy) + } + By("Upgrading providers to the latest version available") clusterctl.UpgradeManagementClusterAndWait(ctx, clusterctl.UpgradeManagementClusterAndWaitInput{ ClusterctlConfigPath: input.ClusterctlConfigPath, @@ -229,6 +238,11 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg By("THE MANAGEMENT CLUSTER WAS SUCCESSFULLY UPGRADED!") + if input.PostUpgrade != nil { + By("Running Post-upgrade steps against the management cluster") + input.PostUpgrade(managementClusterProxy) + } + // After upgrading we are sure the version is the latest version of the API, // so it is possible to use the standard helpers @@ -263,6 +277,23 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg }) if !input.SkipCleanup { + switch { + case discovery.ServerSupportsVersion(managementClusterProxy.GetClientSet().DiscoveryClient, clusterv1.GroupVersion) == nil: + Byf("Deleting all clusters in namespace: %s in management cluster: %s", testNamespace.Name, managementClusterName) + framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{ + Client: managementClusterProxy.GetClient(), + Namespace: testNamespace.Name, + }, input.E2EConfig.GetIntervals(specName, "wait-delete-cluster")...) + case discovery.ServerSupportsVersion(managementClusterProxy.GetClientSet().DiscoveryClient, clusterv1old.GroupVersion) == nil: + Byf("Deleting all clusters in namespace: %s in management cluster: %s", testNamespace.Name, managementClusterName) + deleteAllClustersAndWaitOldAPI(ctx, framework.DeleteAllClustersAndWaitInput{ + Client: managementClusterProxy.GetClient(), + Namespace: testNamespace.Name, + }, input.E2EConfig.GetIntervals(specName, "wait-delete-cluster")...) + default: + log.Logf("Management Cluster does not appear to support CAPI resources.") + } + Byf("Deleting cluster %s and %s", testNamespace.Name, managementClusterName) framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{ Client: managementClusterProxy.GetClient(), @@ -299,3 +330,73 @@ func downloadToTmpFile(url string) string { return tmpFile.Name() } + +// deleteAllClustersAndWaitOldAPI deletes all v1alpha3 cluster resources in the given namespace and waits for them to be gone. +func deleteAllClustersAndWaitOldAPI(ctx context.Context, input framework.DeleteAllClustersAndWaitInput, intervals ...interface{}) { + Expect(ctx).NotTo(BeNil(), "ctx is required for DeleteAllClustersAndWait") + Expect(input.Client).ToNot(BeNil(), "Invalid argument. input.Client can't be nil when calling DeleteAllClustersAndWait") + Expect(input.Namespace).ToNot(BeEmpty(), "Invalid argument. input.Namespace can't be empty when calling DeleteAllClustersAndWait") + + clusters := getAllClustersByNamespaceOldAPI(ctx, framework.GetAllClustersByNamespaceInput{ + Lister: input.Client, + Namespace: input.Namespace, + }) + + for _, c := range clusters { + deleteClusterOldAPI(ctx, deleteClusterOldAPIInput{ + Deleter: input.Client, + Cluster: c, + }) + } + + for _, c := range clusters { + log.Logf("Waiting for the Cluster %s/%s to be deleted", c.Namespace, c.Name) + waitForClusterDeletedOldAPI(ctx, waitForClusterDeletedOldAPIInput{ + Getter: input.Client, + Cluster: c, + }, intervals...) + } +} + +// getAllClustersByNamespaceOldAPI returns the list of v1alpha3 Cluster objects in a namespace. +func getAllClustersByNamespaceOldAPI(ctx context.Context, input framework.GetAllClustersByNamespaceInput) []*clusterv1old.Cluster { + clusterList := &clusterv1old.ClusterList{} + Expect(input.Lister.List(ctx, clusterList, client.InNamespace(input.Namespace))).To(Succeed(), "Failed to list clusters in namespace %s", input.Namespace) + + clusters := make([]*clusterv1old.Cluster, len(clusterList.Items)) + for i := range clusterList.Items { + clusters[i] = &clusterList.Items[i] + } + return clusters +} + +// deleteClusterOldAPIInput is the input for deleteCluster. +type deleteClusterOldAPIInput struct { + Deleter framework.Deleter + Cluster *clusterv1old.Cluster +} + +// deleteClusterOldAPI deletes the v1alpha3 cluster and waits for everything the cluster owned to actually be gone. +func deleteClusterOldAPI(ctx context.Context, input deleteClusterOldAPIInput) { + By(fmt.Sprintf("Deleting cluster %s", input.Cluster.GetName())) + Expect(input.Deleter.Delete(ctx, input.Cluster)).To(Succeed()) +} + +// waitForClusterDeletedOldAPIInput is the input for waitForClusterDeleted. +type waitForClusterDeletedOldAPIInput struct { + Getter framework.Getter + Cluster *clusterv1old.Cluster +} + +// waitForClusterDeletedOldAPI waits until the v1alpha3 cluster object has been deleted. +func waitForClusterDeletedOldAPI(ctx context.Context, input waitForClusterDeletedOldAPIInput, intervals ...interface{}) { + By(fmt.Sprintf("Waiting for cluster %s to be deleted", input.Cluster.GetName())) + Eventually(func() bool { + cluster := &clusterv1old.Cluster{} + key := client.ObjectKey{ + Namespace: input.Cluster.GetNamespace(), + Name: input.Cluster.GetName(), + } + return apierrors.IsNotFound(input.Getter.Get(ctx, key, cluster)) + }, intervals...).Should(BeTrue()) +}