retryable errors in test/framework

test/framework/cluster_helpers.go

@@ -87,7 +87,9 @@ func GetClusterByName(ctx context.Context, input GetClusterByNameInput) *cluster
 		Namespace: input.Namespace,
 		Name:      input.Name,
 	}
-	Expect(input.Getter.Get(ctx, key, cluster)).To(Succeed(), "Failed to get Cluster object %s/%s", input.Namespace, input.Name)
+	Eventually(func() error {
+		return input.Getter.Get(ctx, key, cluster)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get Cluster object %s/%s", input.Namespace, input.Name)
 	return cluster
 }
 
@@ -109,7 +111,9 @@ func PatchClusterLabel(ctx context.Context, input PatchClusterLabelInput) {
 	patchHelper, err := patch.NewHelper(input.Cluster, input.ClusterProxy.GetClient())
 	Expect(err).ToNot(HaveOccurred())
 	input.Cluster.SetLabels(input.Labels)
-	Expect(patchHelper.Patch(ctx, input.Cluster)).To(Succeed())
+	Eventually(func() error {
+		return patchHelper.Patch(ctx, input.Cluster)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 }
 
 // WaitForClusterToProvisionInput is the input for WaitForClusterToProvision.

test/framework/cluster_proxy.go

@@ -309,7 +309,9 @@ func (p *clusterProxy) getKubeconfig(ctx context.Context, namespace string, name
 		Name:      fmt.Sprintf("%s-kubeconfig", name),
 		Namespace: namespace,
 	}
-	Expect(cl.Get(ctx, key, secret)).To(Succeed(), "Failed to get %s", key)
+	Eventually(func() error {
+		return cl.Get(ctx, key, secret)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get %s", key)
 	Expect(secret.Data).To(HaveKey("value"), "Invalid secret %s", key)
 
 	config, err := clientcmd.Load(secret.Data["value"])
@@ -326,7 +328,9 @@ func (p *clusterProxy) isDockerCluster(ctx context.Context, namespace string, na
 		Name:      name,
 		Namespace: namespace,
 	}
-	Expect(cl.Get(ctx, key, cluster)).To(Succeed(), "Failed to get %s", key)
+	Eventually(func() error {
+		return cl.Get(ctx, key, cluster)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get %s", key)
 
 	return cluster.Spec.InfrastructureRef.Kind == "DockerCluster"
 }

test/framework/cluster_topology_helpers.go

@@ -50,7 +50,9 @@ func GetClusterClassByName(ctx context.Context, input GetClusterClassByNameInput
 		Namespace: input.Namespace,
 		Name:      input.Name,
 	}
-	Expect(input.Getter.Get(ctx, key, clusterClass)).To(Succeed(), "Failed to get ClusterClass object %s/%s", input.Namespace, input.Name)
+	Eventually(func() error {
+		return input.Getter.Get(ctx, key, clusterClass)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get ClusterClass object %s/%s", input.Namespace, input.Name)
 	return clusterClass
 }
 
@@ -96,7 +98,9 @@ func UpgradeClusterTopologyAndWaitForUpgrade(ctx context.Context, input UpgradeC
 			input.Cluster.Spec.Topology.Variables[i].Value = apiextensionsv1.JSON{Raw: []byte(strconv.Quote(input.DNSImageTag))}
 		}
 	}
-	Expect(patchHelper.Patch(ctx, input.Cluster)).To(Succeed())
+	Eventually(func() error {
+		return patchHelper.Patch(ctx, input.Cluster)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 	log.Logf("Waiting for control-plane machines to have the upgraded Kubernetes version")
 	WaitForControlPlaneMachinesToBeUpgraded(ctx, WaitForControlPlaneMachinesToBeUpgradedInput{

test/framework/clusterresourceset_helpers.go

@@ -58,7 +58,9 @@ type GetClusterResourceSetBindingByClusterInput struct {
 // GetClusterResourceSetBindingByCluster returns the ClusterResourceBinding objects for a cluster.
 func GetClusterResourceSetBindingByCluster(ctx context.Context, input GetClusterResourceSetBindingByClusterInput) *addonsv1.ClusterResourceSetBinding {
 	binding := &addonsv1.ClusterResourceSetBinding{}
-	Expect(input.Getter.Get(ctx, client.ObjectKey{Namespace: input.Namespace, Name: input.ClusterName}, binding)).To(Succeed(), "Failed to list MachineDeployments object for Cluster %s/%s", input.Namespace, input.ClusterName)
+	Eventually(func() error {
+		return input.Getter.Get(ctx, client.ObjectKey{Namespace: input.Namespace, Name: input.ClusterName}, binding)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list ClusterResourceSetBinding objects for Cluster %s/%s", input.Namespace, input.ClusterName)
 	return binding
 }
 

test/framework/controlplane_helpers.go

@@ -45,7 +45,9 @@ type CreateKubeadmControlPlaneInput struct {
 // CreateKubeadmControlPlane creates the control plane object and necessary dependencies.
 func CreateKubeadmControlPlane(ctx context.Context, input CreateKubeadmControlPlaneInput, intervals ...interface{}) {
 	By("creating the machine template")
-	Expect(input.Creator.Create(ctx, input.MachineTemplate)).To(Succeed())
+	Eventually(func() error {
+		return input.Creator.Create(ctx, input.MachineTemplate)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 	By("creating a KubeadmControlPlane")
 	Eventually(func() error {
@@ -326,7 +328,9 @@ func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeCont
 	input.ControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local.ImageMeta.ImageTag = input.EtcdImageTag
 	input.ControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.DNS.ImageMeta.ImageTag = input.DNSImageTag
 
-	Expect(patchHelper.Patch(ctx, input.ControlPlane)).To(Succeed())
+	Eventually(func() error {
+		return patchHelper.Patch(ctx, input.ControlPlane)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 	log.Logf("Waiting for control-plane machines to have the upgraded kubernetes version")
 	WaitForControlPlaneMachinesToBeUpgraded(ctx, WaitForControlPlaneMachinesToBeUpgradedInput{
@@ -386,7 +390,9 @@ func ScaleAndWaitControlPlane(ctx context.Context, input ScaleAndWaitControlPlan
 	scaleBefore := pointer.Int32Deref(input.ControlPlane.Spec.Replicas, 0)
 	input.ControlPlane.Spec.Replicas = pointer.Int32Ptr(input.Replicas)
 	log.Logf("Scaling controlplane %s/%s from %v to %v replicas", input.ControlPlane.Namespace, input.ControlPlane.Name, scaleBefore, input.Replicas)
-	Expect(patchHelper.Patch(ctx, input.ControlPlane)).To(Succeed())
+	Eventually(func() error {
+		return patchHelper.Patch(ctx, input.ControlPlane)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 	log.Logf("Waiting for correct number of replicas to exist")
 	Eventually(func() (int, error) {

test/framework/deployment_helpers.go

@@ -32,6 +32,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/api/policy/v1beta1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	utilversion "k8s.io/apimachinery/pkg/util/version"
@@ -109,7 +110,9 @@ func WatchDeploymentLogs(ctx context.Context, input WatchDeploymentLogsInput) {
 
 	deployment := &appsv1.Deployment{}
 	key := client.ObjectKeyFromObject(input.Deployment)
-	Expect(input.GetLister.Get(ctx, key, deployment)).To(Succeed(), "Failed to get deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
+	Eventually(func() error {
+		return input.GetLister.Get(ctx, key, deployment)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
 
 	selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector)
 	Expect(err).NotTo(HaveOccurred(), "Failed to Pods selector for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
@@ -174,7 +177,9 @@ func WatchPodMetrics(ctx context.Context, input WatchPodMetricsInput) {
 
 	deployment := &appsv1.Deployment{}
 	key := client.ObjectKeyFromObject(input.Deployment)
-	Expect(input.GetLister.Get(ctx, key, deployment)).To(Succeed(), "Failed to get deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
+	Eventually(func() error {
+		return input.GetLister.Get(ctx, key, deployment)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
 
 	selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector)
 	Expect(err).NotTo(HaveOccurred(), "Failed to Pods selector for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name)
@@ -266,14 +271,21 @@ func DeployUnevictablePod(ctx context.Context, input DeployUnevictablePodInput)
 	workloadClient := input.WorkloadClusterProxy.GetClientSet()
 
 	log.Logf("Check if namespace %s exists", input.Namespace)
-	if _, err := workloadClient.CoreV1().Namespaces().Get(ctx, input.Namespace, metav1.GetOptions{}); err != nil {
-		_, errCreateNamespace := workloadClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: input.Namespace,
-			},
-		}, metav1.CreateOptions{})
-		Expect(errCreateNamespace).To(BeNil())
-	}
+	Eventually(func() error {
+		_, err := workloadClient.CoreV1().Namespaces().Get(ctx, input.Namespace, metav1.GetOptions{})
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				_, errCreateNamespace := workloadClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: input.Namespace,
+					},
+				}, metav1.CreateOptions{})
+				return errCreateNamespace
+			}
+			return err
+		}
+		return nil
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 	workloadDeployment := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
@@ -380,9 +392,13 @@ type AddDeploymentToWorkloadClusterInput struct {
 }
 
 func AddDeploymentToWorkloadCluster(ctx context.Context, input AddDeploymentToWorkloadClusterInput) {
-	result, err := input.ClientSet.AppsV1().Deployments(input.Namespace).Create(ctx, input.Deployment, metav1.CreateOptions{})
-	Expect(result).NotTo(BeNil())
-	Expect(err).To(BeNil(), "nonstop pods need to be successfully deployed")
+	Eventually(func() error {
+		result, err := input.ClientSet.AppsV1().Deployments(input.Namespace).Create(ctx, input.Deployment, metav1.CreateOptions{})
+		if result != nil && err == nil {
+			return nil
+		}
+		return fmt.Errorf("deployment %s in namespace %s not successfully created in workload cluster: %v", input.Deployment.Name, input.Namespace, err)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 }
 
 type AddPodDisruptionBudgetInput struct {

test/framework/machine_helpers.go

@@ -204,11 +204,15 @@ func PatchNodeCondition(ctx context.Context, input PatchNodeConditionInput) {
 	log.Logf("Patching the node condition to the node")
 	Expect(input.Machine.Status.NodeRef).ToNot(BeNil())
 	node := &corev1.Node{}
-	Expect(input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name).GetClient().Get(ctx, types.NamespacedName{Name: input.Machine.Status.NodeRef.Name, Namespace: input.Machine.Status.NodeRef.Namespace}, node)).To(Succeed())
+	Eventually(func() error {
+		return input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name).GetClient().Get(ctx, types.NamespacedName{Name: input.Machine.Status.NodeRef.Name, Namespace: input.Machine.Status.NodeRef.Namespace}, node)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 	patchHelper, err := patch.NewHelper(node, input.ClusterProxy.GetWorkloadCluster(ctx, input.Cluster.Namespace, input.Cluster.Name).GetClient())
 	Expect(err).ToNot(HaveOccurred())
 	node.Status.Conditions = append(node.Status.Conditions, input.NodeCondition)
-	Expect(patchHelper.Patch(ctx, node)).To(Succeed())
+	Eventually(func() error {
+		return patchHelper.Patch(ctx, node)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 }
 
 // MachineStatusCheck is a type that operates a status check on a Machine.

test/framework/machinedeployment_helpers.go

@@ -45,13 +45,19 @@ type CreateMachineDeploymentInput struct {
 // CreateMachineDeployment creates the machine deployment and dependencies.
 func CreateMachineDeployment(ctx context.Context, input CreateMachineDeploymentInput) {
 	By("creating a core MachineDeployment resource")
-	Expect(input.Creator.Create(ctx, input.MachineDeployment)).To(Succeed())
+	Eventually(func() error {
+		return input.Creator.Create(ctx, input.MachineDeployment)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 	By("creating a BootstrapConfigTemplate resource")
-	Expect(input.Creator.Create(ctx, input.BootstrapConfigTemplate)).To(Succeed())
+	Eventually(func() error {
+		return input.Creator.Create(ctx, input.BootstrapConfigTemplate)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 	By("creating an InfrastructureMachineTemplate resource")
-	Expect(input.Creator.Create(ctx, input.InfraMachineTemplate)).To(Succeed())
+	Eventually(func() error {
+		return input.Creator.Create(ctx, input.InfraMachineTemplate)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 }
 
 // GetMachineDeploymentsByClusterInput is the input for GetMachineDeploymentsByCluster.
@@ -178,7 +184,9 @@ func UpgradeMachineDeploymentsAndWait(ctx context.Context, input UpgradeMachineD
 		if input.UpgradeMachineTemplate != nil {
 			deployment.Spec.Template.Spec.InfrastructureRef.Name = *input.UpgradeMachineTemplate
 		}
-		Expect(patchHelper.Patch(ctx, deployment)).To(Succeed())
+		Eventually(func() error {
+			return patchHelper.Patch(ctx, deployment)
+		}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 		log.Logf("Waiting for Kubernetes versions of machines in MachineDeployment %s/%s to be upgraded from %s to %s",
 			deployment.Namespace, deployment.Name, *oldVersion, input.UpgradeVersion)
@@ -259,21 +267,27 @@ func UpgradeMachineDeploymentInfrastructureRefAndWait(ctx context.Context, input
 			Namespace: input.Cluster.Namespace,
 			Name:      infraRef.Name,
 		}
-		Expect(mgmtClient.Get(ctx, key, infraObj)).NotTo(HaveOccurred())
+		Eventually(func() error {
+			return mgmtClient.Get(ctx, key, infraObj)
+		}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 		// Creates a new infra object
 		newInfraObj := infraObj
 		newInfraObjName := fmt.Sprintf("%s-%s", infraRef.Name, util.RandomString(6))
 		newInfraObj.SetName(newInfraObjName)
 		newInfraObj.SetResourceVersion("")
-		Expect(mgmtClient.Create(ctx, newInfraObj)).NotTo(HaveOccurred())
+		Eventually(func() error {
+			return mgmtClient.Create(ctx, newInfraObj)
+		}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 		// Patch the new infra object's ref to the machine deployment
 		patchHelper, err := patch.NewHelper(deployment, mgmtClient)
 		Expect(err).ToNot(HaveOccurred())
 		infraRef.Name = newInfraObjName
 		deployment.Spec.Template.Spec.InfrastructureRef = infraRef
-		Expect(patchHelper.Patch(ctx, deployment)).To(Succeed())
+		Eventually(func() error {
+			return patchHelper.Patch(ctx, deployment)
+		}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 		log.Logf("Waiting for rolling upgrade to start.")
 		WaitForMachineDeploymentRollingUpgradeToStart(ctx, WaitForMachineDeploymentRollingUpgradeToStartInput{
@@ -315,7 +329,9 @@ func ScaleAndWaitMachineDeployment(ctx context.Context, input ScaleAndWaitMachin
 	patchHelper, err := patch.NewHelper(input.MachineDeployment, input.ClusterProxy.GetClient())
 	Expect(err).ToNot(HaveOccurred())
 	input.MachineDeployment.Spec.Replicas = pointer.Int32Ptr(input.Replicas)
-	Expect(patchHelper.Patch(ctx, input.MachineDeployment)).To(Succeed())
+	Eventually(func() error {
+		return patchHelper.Patch(ctx, input.MachineDeployment)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 
 	log.Logf("Waiting for correct number of replicas to exist")
 	Eventually(func() (int, error) {

test/framework/machinepool_helpers.go

@@ -199,7 +199,9 @@ func ScaleMachinePoolAndWait(ctx context.Context, input ScaleMachinePoolAndWaitI
 		Expect(err).ToNot(HaveOccurred())
 
 		mp.Spec.Replicas = &input.Replicas
-		Expect(patchHelper.Patch(ctx, mp)).To(Succeed())
+		Eventually(func() error {
+			return patchHelper.Patch(ctx, mp)
+		}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 	}
 
 	for _, mp := range input.MachinePools {

test/framework/namespace_helpers.go

@@ -77,7 +77,9 @@ func EnsureNamespace(ctx context.Context, mgmt client.Client, namespace string)
 				Name: namespace,
 			},
 		}
-		Expect(mgmt.Create(ctx, ns)).To(Succeed())
+		Eventually(func() error {
+			return mgmt.Create(ctx, ns)
+		}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed())
 	} else {
 		Fail(err.Error())
 	}