[e2e framework] Add ability to run pre and post actions during cluste…

…rctl upgrade spec

Also enable better cleanup in clusterctl upgrade spec when management
cluster hasn't been upgraded yet.

test/e2e/clusterctl_upgrade.go

@@ -30,7 +30,9 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/discovery"
 	"k8s.io/utils/pointer"
 	clusterv1old "sigs.k8s.io/cluster-api/api/v1alpha3"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
@@ -52,6 +54,8 @@ type ClusterctlUpgradeSpecInput struct {
 	BootstrapClusterProxy framework.ClusterProxy
 	ArtifactFolder        string
 	SkipCleanup           bool
+	PreUpgrade            func(managementClusterProxy framework.ClusterProxy)
+	PostUpgrade           func(managementClusterProxy framework.ClusterProxy)
 }
 
 // ClusterctlUpgradeSpec implements a test that verifies clusterctl upgrade of a management cluster.
@@ -219,6 +223,11 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
 
 		By("THE MANAGEMENT CLUSTER WITH OLDER VERSION OF PROVIDERS WORKS!")
 
+		if input.PreUpgrade != nil {
+			By("Running Pre-upgrade steps against the management cluster")
+			input.PreUpgrade(managementClusterProxy)
+		}
+
 		By("Upgrading providers to the latest version available")
 		clusterctl.UpgradeManagementClusterAndWait(ctx, clusterctl.UpgradeManagementClusterAndWaitInput{
 			ClusterctlConfigPath: input.ClusterctlConfigPath,
@@ -229,6 +238,11 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
 
 		By("THE MANAGEMENT CLUSTER WAS SUCCESSFULLY UPGRADED!")
 
+		if input.PostUpgrade != nil {
+			By("Running Post-upgrade steps against the management cluster")
+			input.PostUpgrade(managementClusterProxy)
+		}
+
 		// After upgrading we are sure the version is the latest version of the API,
 		// so it is possible to use the standard helpers
 
@@ -263,6 +277,23 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
 			})
 
 			if !input.SkipCleanup {
+				switch {
+				case discovery.ServerSupportsVersion(managementClusterProxy.GetClientSet().DiscoveryClient, clusterv1.GroupVersion) == nil:
+					Byf("Deleting all clusters in namespace: %s in management cluster: %s", testNamespace.Name, managementClusterName)
+					framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{
+						Client:    managementClusterProxy.GetClient(),
+						Namespace: testNamespace.Name,
+					}, input.E2EConfig.GetIntervals(specName, "wait-delete-cluster")...)
+				case discovery.ServerSupportsVersion(managementClusterProxy.GetClientSet().DiscoveryClient, clusterv1old.GroupVersion) == nil:
+					Byf("Deleting all clusters in namespace: %s in management cluster: %s", testNamespace.Name, managementClusterName)
+					deleteAllClustersAndWaitOldAPI(ctx, framework.DeleteAllClustersAndWaitInput{
+						Client:    managementClusterProxy.GetClient(),
+						Namespace: testNamespace.Name,
+					}, input.E2EConfig.GetIntervals(specName, "wait-delete-cluster")...)
+				default:
+					log.Logf("Management Cluster does not appear to support CAPI resources.")
+				}
+
 				Byf("Deleting cluster %s and %s", testNamespace.Name, managementClusterName)
 				framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{
 					Client:    managementClusterProxy.GetClient(),
@@ -299,3 +330,73 @@ func downloadToTmpFile(url string) string {
 
 	return tmpFile.Name()
 }
+
+// deleteAllClustersAndWaitOldAPI deletes all v1alpha3 cluster resources in the given namespace and waits for them to be gone.
+func deleteAllClustersAndWaitOldAPI(ctx context.Context, input framework.DeleteAllClustersAndWaitInput, intervals ...interface{}) {
+	Expect(ctx).NotTo(BeNil(), "ctx is required for DeleteAllClustersAndWait")
+	Expect(input.Client).ToNot(BeNil(), "Invalid argument. input.Client can't be nil when calling DeleteAllClustersAndWait")
+	Expect(input.Namespace).ToNot(BeEmpty(), "Invalid argument. input.Namespace can't be empty when calling DeleteAllClustersAndWait")
+
+	clusters := getAllClustersByNamespaceOldAPI(ctx, framework.GetAllClustersByNamespaceInput{
+		Lister:    input.Client,
+		Namespace: input.Namespace,
+	})
+
+	for _, c := range clusters {
+		deleteClusterOldAPI(ctx, deleteClusterOldAPIInput{
+			Deleter: input.Client,
+			Cluster: c,
+		})
+	}
+
+	for _, c := range clusters {
+		log.Logf("Waiting for the Cluster %s/%s to be deleted", c.Namespace, c.Name)
+		waitForClusterDeletedOldAPI(ctx, waitForClusterDeletedOldAPIInput{
+			Getter:  input.Client,
+			Cluster: c,
+		}, intervals...)
+	}
+}
+
+// getAllClustersByNamespaceOldAPI returns the list of v1alpha3 Cluster objects in a namespace.
+func getAllClustersByNamespaceOldAPI(ctx context.Context, input framework.GetAllClustersByNamespaceInput) []*clusterv1old.Cluster {
+	clusterList := &clusterv1old.ClusterList{}
+	Expect(input.Lister.List(ctx, clusterList, client.InNamespace(input.Namespace))).To(Succeed(), "Failed to list clusters in namespace %s", input.Namespace)
+
+	clusters := make([]*clusterv1old.Cluster, len(clusterList.Items))
+	for i := range clusterList.Items {
+		clusters[i] = &clusterList.Items[i]
+	}
+	return clusters
+}
+
+// deleteClusterOldAPIInput is the input for deleteCluster.
+type deleteClusterOldAPIInput struct {
+	Deleter framework.Deleter
+	Cluster *clusterv1old.Cluster
+}
+
+// deleteClusterOldAPI deletes the v1alpha3 cluster and waits for everything the cluster owned to actually be gone.
+func deleteClusterOldAPI(ctx context.Context, input deleteClusterOldAPIInput) {
+	By(fmt.Sprintf("Deleting cluster %s", input.Cluster.GetName()))
+	Expect(input.Deleter.Delete(ctx, input.Cluster)).To(Succeed())
+}
+
+// waitForClusterDeletedOldAPIInput is the input for waitForClusterDeleted.
+type waitForClusterDeletedOldAPIInput struct {
+	Getter  framework.Getter
+	Cluster *clusterv1old.Cluster
+}
+
+// waitForClusterDeletedOldAPI waits until the v1alpha3 cluster object has been deleted.
+func waitForClusterDeletedOldAPI(ctx context.Context, input waitForClusterDeletedOldAPIInput, intervals ...interface{}) {
+	By(fmt.Sprintf("Waiting for cluster %s to be deleted", input.Cluster.GetName()))
+	Eventually(func() bool {
+		cluster := &clusterv1old.Cluster{}
+		key := client.ObjectKey{
+			Namespace: input.Cluster.GetNamespace(),
+			Name:      input.Cluster.GetName(),
+		}
+		return apierrors.IsNotFound(input.Getter.Get(ctx, key, cluster))
+	}, intervals...).Should(BeTrue())
+}