more retryable errors in test/framework

test/framework/alltypes_helpers.go

@@ -81,8 +81,9 @@ func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta {
 	discoveredTypes := []metav1.TypeMeta{}
 
 	crdList := &apiextensionsv1.CustomResourceDefinitionList{}
-	err := lister.List(ctx, crdList, capiProviderOptions()...)
-	Expect(err).ToNot(HaveOccurred(), "failed to list CRDs for CAPI providers")
+	Eventually(func() error {
+		return lister.List(ctx, crdList, capiProviderOptions()...)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "failed to list CRDs for CAPI providers")
 
 	for _, crd := range crdList.Items {
 		for _, version := range crd.Spec.Versions {

test/framework/cluster_helpers.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
@@ -40,7 +41,9 @@ type CreateClusterInput struct {
 // CreateCluster will create the Cluster and InfraCluster objects.
 func CreateCluster(ctx context.Context, input CreateClusterInput, intervals ...interface{}) {
 	By("creating an InfrastructureCluster resource")
-	Expect(input.Creator.Create(ctx, input.InfraCluster)).To(Succeed())
+	Eventually(func() error {
+		return input.Creator.Create(ctx, input.InfraCluster)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 	// This call happens in an eventually because of a race condition with the
 	// webhook server. If the latter isn't fully online then this call will
@@ -64,7 +67,9 @@ type GetAllClustersByNamespaceInput struct {
 // GetAllClustersByNamespace returns the list of Cluster object in a namespace.
 func GetAllClustersByNamespace(ctx context.Context, input GetAllClustersByNamespaceInput) []*clusterv1.Cluster {
 	clusterList := &clusterv1.ClusterList{}
-	Expect(input.Lister.List(ctx, clusterList, client.InNamespace(input.Namespace))).To(Succeed(), "Failed to list clusters in namespace %s", input.Namespace)
+	Eventually(func() error {
+		return input.Lister.List(ctx, clusterList, client.InNamespace(input.Namespace))
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list clusters in namespace %s", input.Namespace)
 
 	clusters := make([]*clusterv1.Cluster, len(clusterList.Items))
 	for i := range clusterList.Items {
@@ -184,12 +189,15 @@ func DiscoveryAndWaitForCluster(ctx context.Context, input DiscoveryAndWaitForCl
 	Expect(input.Namespace).ToNot(BeNil(), "Invalid argument. input.Namespace can't be empty when calling DiscoveryAndWaitForCluster")
 	Expect(input.Name).ToNot(BeNil(), "Invalid argument. input.Name can't be empty when calling DiscoveryAndWaitForCluster")
 
-	cluster := GetClusterByName(ctx, GetClusterByNameInput{
-		Getter:    input.Getter,
-		Name:      input.Name,
-		Namespace: input.Namespace,
-	})
-	Expect(cluster).ToNot(BeNil(), "Failed to get the Cluster object")
+	var cluster *clusterv1.Cluster
+	Eventually(func() bool {
+		cluster = GetClusterByName(ctx, GetClusterByNameInput{
+			Getter:    input.Getter,
+			Name:      input.Name,
+			Namespace: input.Namespace,
+		})
+		return cluster != nil
+	}, retryableOperationTimeout, retryableOperationInterval).Should(BeTrue(), "Failed to get Cluster object %s/%s", input.Namespace, input.Name)
 
 	// NOTE: We intentionally return the provisioned Cluster because it also contains
 	// the reconciled ControlPlane ref and InfrastructureCluster ref when using a ClusterClass.
@@ -226,11 +234,12 @@ func DeleteClusterAndWait(ctx context.Context, input DeleteClusterAndWaitInput,
 
 	//TODO: consider if to move in another func (what if there are more than one cluster?)
 	log.Logf("Check for all the Cluster API resources being deleted")
-	resources := GetCAPIResources(ctx, GetCAPIResourcesInput{
-		Lister:    input.Client,
-		Namespace: input.Cluster.Namespace,
-	})
-	Expect(resources).To(BeEmpty(), "There are still Cluster API resources in the %q namespace", input.Cluster.Namespace)
+	Eventually(func() []*unstructured.Unstructured {
+		return GetCAPIResources(ctx, GetCAPIResourcesInput{
+			Lister:    input.Client,
+			Namespace: input.Cluster.Namespace,
+		})
+	}, retryableOperationTimeout, retryableOperationInterval).Should(BeEmpty(), "There are still Cluster API resources in the %q namespace", input.Cluster.Namespace)
 }
 
 // DeleteAllClustersAndWaitInput is the input type for DeleteAllClustersAndWait.

test/framework/cluster_proxy.go

@@ -248,8 +248,12 @@ func (p *clusterProxy) CollectWorkloadClusterLogs(ctx context.Context, namespace
 		return
 	}
 
-	machines, err := getMachinesInCluster(ctx, p.GetClient(), namespace, name)
-	Expect(err).ToNot(HaveOccurred(), "Failed to get machines for the %s/%s cluster", namespace, name)
+	var machines *clusterv1.MachineList
+	Eventually(func() error {
+		var err error
+		machines, err = getMachinesInCluster(ctx, p.GetClient(), namespace, name)
+		return err
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get machines for the %s/%s cluster", namespace, name)
 
 	for i := range machines.Items {
 		m := &machines.Items[i]
@@ -260,8 +264,12 @@ func (p *clusterProxy) CollectWorkloadClusterLogs(ctx context.Context, namespace
 		}
 	}
 
-	machinePools, err := getMachinePoolsInCluster(ctx, p.GetClient(), namespace, name)
-	Expect(err).ToNot(HaveOccurred(), "Failed to get machine pools for the %s/%s cluster", namespace, name)
+	var machinePools *expv1.MachinePoolList
+	Eventually(func() error {
+		var err error
+		machinePools, err = getMachinePoolsInCluster(ctx, p.GetClient(), namespace, name)
+		return err
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get machine pools for the %s/%s cluster", namespace, name)
 
 	for i := range machinePools.Items {
 		mp := &machinePools.Items[i]

test/framework/clusterresourceset_helpers.go

@@ -39,7 +39,9 @@ type GetClusterResourceSetsInput struct {
 // GetClusterResourceSets returns all ClusterResourceSet objects in a namespace.
 func GetClusterResourceSets(ctx context.Context, input GetClusterResourceSetsInput) []*addonsv1.ClusterResourceSet {
 	crsList := &addonsv1.ClusterResourceSetList{}
-	Expect(input.Lister.List(ctx, crsList, client.InNamespace(input.Namespace))).To(Succeed(), "Failed to list ClusterResourceSet objects for namespace %s", input.Namespace)
+	Eventually(func() error {
+		return input.Lister.List(ctx, crsList, client.InNamespace(input.Namespace))
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list ClusterResourceSet objects for namespace %s", input.Namespace)
 
 	clusterResourceSets := make([]*addonsv1.ClusterResourceSet, len(crsList.Items))
 	for i := range crsList.Items {
@@ -78,12 +80,14 @@ func DiscoverClusterResourceSetAndWaitForSuccess(ctx context.Context, input Disc
 
 	mgmtClient := input.ClusterProxy.GetClient()
 	fmt.Fprintln(GinkgoWriter, "Discovering cluster resource set resources")
-	clusterResourceSets := GetClusterResourceSets(ctx, GetClusterResourceSetsInput{
-		Lister:    mgmtClient,
-		Namespace: input.Cluster.Namespace,
-	})
-
-	Expect(clusterResourceSets).NotTo(BeEmpty())
+	var clusterResourceSets []*addonsv1.ClusterResourceSet
+	Eventually(func() bool {
+		clusterResourceSets = GetClusterResourceSets(ctx, GetClusterResourceSetsInput{
+			Lister:    mgmtClient,
+			Namespace: input.Cluster.Namespace,
+		})
+		return len(clusterResourceSets) > 0
+	}, retryableOperationTimeout, retryableOperationInterval).Should(BeTrue())
 
 	for _, crs := range clusterResourceSets {
 		Expect(crs.Spec.ClusterSelector).NotTo(BeNil())

test/framework/controller_helpers.go

@@ -32,7 +32,9 @@ type GetControllerDeploymentsInput struct {
 // GetControllerDeployments returns all the deployment for the cluster API controllers existing in a management cluster.
 func GetControllerDeployments(ctx context.Context, input GetControllerDeploymentsInput) []*appsv1.Deployment {
 	deploymentList := &appsv1.DeploymentList{}
-	Expect(input.Lister.List(ctx, deploymentList, capiProviderOptions()...)).To(Succeed(), "Failed to list deployments for the cluster API controllers")
+	Eventually(func() error {
+		return input.Lister.List(ctx, deploymentList, capiProviderOptions()...)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list deployments for the cluster API controllers")
 
 	deployments := make([]*appsv1.Deployment, 0, len(deploymentList.Items))
 	for i := range deploymentList.Items {

test/framework/controlplane_helpers.go

@@ -71,7 +71,9 @@ type GetKubeadmControlPlaneByClusterInput struct {
 // it is necessary to ensure this is already happened before calling it.
 func GetKubeadmControlPlaneByCluster(ctx context.Context, input GetKubeadmControlPlaneByClusterInput) *controlplanev1.KubeadmControlPlane {
 	controlPlaneList := &controlplanev1.KubeadmControlPlaneList{}
-	Expect(input.Lister.List(ctx, controlPlaneList, byClusterOptions(input.ClusterName, input.Namespace)...)).To(Succeed(), "Failed to list KubeadmControlPlane object for Cluster %s/%s", input.Namespace, input.ClusterName)
+	Eventually(func() error {
+		return input.Lister.List(ctx, controlPlaneList, byClusterOptions(input.ClusterName, input.Namespace)...)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list KubeadmControlPlane object for Cluster %s/%s", input.Namespace, input.ClusterName)
 	Expect(len(controlPlaneList.Items)).ToNot(BeNumerically(">", 1), "Cluster %s/%s should not have more than 1 KubeadmControlPlane object", input.Namespace, input.ClusterName)
 	if len(controlPlaneList.Items) == 1 {
 		return &controlPlaneList.Items[0]
@@ -205,8 +207,9 @@ func AssertControlPlaneFailureDomains(ctx context.Context, input AssertControlPl
 	}
 
 	machineList := &clusterv1.MachineList{}
-	Expect(input.GetLister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption)).
-		To(Succeed(), "Couldn't list machines for the cluster %q", input.ClusterKey.Name)
+	Eventually(func() error {
+		return input.GetLister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Couldn't list machines for the cluster %q", input.ClusterKey.Name)
 
 	// Count all control plane machine failure domains.
 	for _, machine := range machineList.Items {

test/framework/deployment_helpers.go

@@ -36,6 +36,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	utilversion "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -185,7 +186,9 @@ func WatchPodMetrics(ctx context.Context, input WatchPodMetricsInput) {
 	Expect(err).NotTo(HaveOccurred(), "Failed to Pods selector for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
 
 	pods := &corev1.PodList{}
-	Expect(input.GetLister.List(ctx, pods, client.InNamespace(input.Deployment.Namespace), client.MatchingLabels(selector))).To(Succeed(), "Failed to list Pods for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
+	Eventually(func() error {
+		return input.GetLister.List(ctx, pods, client.InNamespace(input.Deployment.Namespace), client.MatchingLabels(selector))
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list Pods for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
 
 	go func() {
 		defer GinkgoRecover()
@@ -324,8 +327,12 @@ func DeployUnevictablePod(ctx context.Context, input DeployUnevictablePodInput)
 		},
 	}
 	if input.ControlPlane != nil {
-		serverVersion, err := workloadClient.ServerVersion()
-		Expect(err).ToNot(HaveOccurred())
+		var serverVersion *version.Info
+		Eventually(func() error {
+			var err error
+			serverVersion, err = workloadClient.ServerVersion()
+			return err
+		}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 		// Use the control-plane label for Kubernetes version >= v1.20.0.
 		if utilversion.MustParseGeneric(serverVersion.String()).AtLeast(utilversion.MustParseGeneric("v1.20.0")) {
@@ -408,7 +415,11 @@ type AddPodDisruptionBudgetInput struct {
 }
 
 func AddPodDisruptionBudget(ctx context.Context, input AddPodDisruptionBudgetInput) {
-	budget, err := input.ClientSet.PolicyV1beta1().PodDisruptionBudgets(input.Namespace).Create(ctx, input.Budget, metav1.CreateOptions{})
-	Expect(budget).NotTo(BeNil())
-	Expect(err).To(BeNil(), "podDisruptionBudget needs to be successfully deployed")
+	Eventually(func() error {
+		budget, err := input.ClientSet.PolicyV1beta1().PodDisruptionBudgets(input.Namespace).Create(ctx, input.Budget, metav1.CreateOptions{})
+		if budget != nil && err == nil {
+			return nil
+		}
+		return fmt.Errorf("podDisruptionBudget needs to be successfully deployed: %v", err)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "podDisruptionBudget needs to be successfully deployed")
 }

test/framework/machine_helpers.go

@@ -55,7 +55,9 @@ func GetMachinesByMachineDeployments(ctx context.Context, input GetMachinesByMac
 	opts = append(opts, machineDeploymentOptions(input.MachineDeployment)...)
 
 	machineList := &clusterv1.MachineList{}
-	Expect(input.Lister.List(ctx, machineList, opts...)).To(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.Namespace, input.ClusterName)
+	Eventually(func() error {
+		return input.Lister.List(ctx, machineList, opts...)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.Namespace, input.ClusterName)
 
 	return machineList.Items
 }
@@ -78,7 +80,9 @@ func GetMachinesByMachineHealthCheck(ctx context.Context, input GetMachinesByMac
 	opts = append(opts, machineHealthCheckOptions(*input.MachineHealthCheck)...)
 
 	machineList := &clusterv1.MachineList{}
-	Expect(input.Lister.List(ctx, machineList, opts...)).To(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.MachineHealthCheck.Namespace, input.ClusterName)
+	Eventually(func() error {
+		return input.Lister.List(ctx, machineList, opts...)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.MachineHealthCheck.Namespace, input.ClusterName)
 
 	return machineList.Items
 }
@@ -102,7 +106,9 @@ func GetControlPlaneMachinesByCluster(ctx context.Context, input GetControlPlane
 	options := append(byClusterOptions(input.ClusterName, input.Namespace), controlPlaneMachineOptions()...)
 
 	machineList := &clusterv1.MachineList{}
-	Expect(input.Lister.List(ctx, machineList, options...)).To(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.Namespace, input.ClusterName)
+	Eventually(func() error {
+		return input.Lister.List(ctx, machineList, options...)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.Namespace, input.ClusterName)
 
 	return machineList.Items
 }

test/framework/machinedeployment_helpers.go

@@ -72,7 +72,9 @@ type GetMachineDeploymentsByClusterInput struct {
 // it is necessary to ensure this is already happened before calling it.
 func GetMachineDeploymentsByCluster(ctx context.Context, input GetMachineDeploymentsByClusterInput) []*clusterv1.MachineDeployment {
 	deploymentList := &clusterv1.MachineDeploymentList{}
-	Expect(input.Lister.List(ctx, deploymentList, byClusterOptions(input.ClusterName, input.Namespace)...)).To(Succeed(), "Failed to list MachineDeployments object for Cluster %s/%s", input.Namespace, input.ClusterName)
+	Eventually(func() error {
+		return input.Lister.List(ctx, deploymentList, byClusterOptions(input.ClusterName, input.Namespace)...)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineDeployments object for Cluster %s/%s", input.Namespace, input.ClusterName)
 
 	deployments := make([]*clusterv1.MachineDeployment, len(deploymentList.Items))
 	for i := range deploymentList.Items {
@@ -213,9 +215,9 @@ func WaitForMachineDeploymentRollingUpgradeToStart(ctx context.Context, input Wa
 	Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling WaitForMachineDeploymentRollingUpgradeToStarts")
 
 	log.Logf("Waiting for MachineDeployment rolling upgrade to start")
-	Eventually(func() bool {
+	Eventually(func(g Gomega) bool {
 		md := &clusterv1.MachineDeployment{}
-		Expect(input.Getter.Get(ctx, client.ObjectKey{Namespace: input.MachineDeployment.Namespace, Name: input.MachineDeployment.Name}, md)).To(Succeed())
+		g.Expect(input.Getter.Get(ctx, client.ObjectKey{Namespace: input.MachineDeployment.Namespace, Name: input.MachineDeployment.Name}, md)).To(Succeed())
 		return md.Status.Replicas != md.Status.AvailableReplicas
 	}, intervals...).Should(BeTrue())
 }
@@ -233,9 +235,9 @@ func WaitForMachineDeploymentRollingUpgradeToComplete(ctx context.Context, input
 	Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling WaitForMachineDeploymentRollingUpgradeToComplete")
 
 	log.Logf("Waiting for MachineDeployment rolling upgrade to complete")
-	Eventually(func() bool {
+	Eventually(func(g Gomega) bool {
 		md := &clusterv1.MachineDeployment{}
-		Expect(input.Getter.Get(ctx, client.ObjectKey{Namespace: input.MachineDeployment.Namespace, Name: input.MachineDeployment.Name}, md)).To(Succeed())
+		g.Expect(input.Getter.Get(ctx, client.ObjectKey{Namespace: input.MachineDeployment.Namespace, Name: input.MachineDeployment.Name}, md)).To(Succeed())
 		return md.Status.Replicas == md.Status.AvailableReplicas
 	}, intervals...).Should(BeTrue())
 }

test/framework/machinehealthcheck_helpers.go

@@ -101,7 +101,9 @@ type GetMachineHealthChecksForClusterInput struct {
 // it is necessary to ensure this is already happened before calling it.
 func GetMachineHealthChecksForCluster(ctx context.Context, input GetMachineHealthChecksForClusterInput) []*clusterv1.MachineHealthCheck {
 	machineHealthCheckList := &clusterv1.MachineHealthCheckList{}
-	Expect(input.Lister.List(ctx, machineHealthCheckList, byClusterOptions(input.ClusterName, input.Namespace)...)).To(Succeed(), "Failed to list MachineDeployments object for Cluster %s/%s", input.Namespace, input.ClusterName)
+	Eventually(func() error {
+		return input.Lister.List(ctx, machineHealthCheckList, byClusterOptions(input.ClusterName, input.Namespace)...)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineHealthCheck object for Cluster %s/%s", input.Namespace, input.ClusterName)
 
 	machineHealthChecks := make([]*clusterv1.MachineHealthCheck, len(machineHealthCheckList.Items))
 	for i := range machineHealthCheckList.Items {