From d777df17e92aa8f13b991cae2c07f51422922827 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 1 Jun 2022 09:26:54 -0700 Subject: [PATCH] more retryable errors in test/framework --- test/framework/alltypes_helpers.go | 5 +-- test/framework/cluster_helpers.go | 35 ++++++++++++-------- test/framework/cluster_proxy.go | 16 ++++++--- test/framework/clusterresourceset_helpers.go | 18 ++++++---- test/framework/controller_helpers.go | 4 ++- test/framework/controlplane_helpers.go | 9 +++-- test/framework/deployment_helpers.go | 23 +++++++++---- test/framework/machine_helpers.go | 12 +++++-- test/framework/machinedeployment_helpers.go | 12 ++++--- test/framework/machinehealthcheck_helpers.go | 4 ++- test/framework/machinepool_helpers.go | 13 ++++++-- test/framework/machines.go | 8 +++-- 12 files changed, 110 insertions(+), 49 deletions(-) diff --git a/test/framework/alltypes_helpers.go b/test/framework/alltypes_helpers.go index 85ef08136323..39d72abac0c1 100644 --- a/test/framework/alltypes_helpers.go +++ b/test/framework/alltypes_helpers.go @@ -81,8 +81,9 @@ func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta { discoveredTypes := []metav1.TypeMeta{} crdList := &apiextensionsv1.CustomResourceDefinitionList{} - err := lister.List(ctx, crdList, capiProviderOptions()...) - Expect(err).ToNot(HaveOccurred(), "failed to list CRDs for CAPI providers") + Eventually(func() error { + return lister.List(ctx, crdList, capiProviderOptions()...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "failed to list CRDs for CAPI providers") for _, crd := range crdList.Items { for _, version := range crd.Spec.Versions { diff --git a/test/framework/cluster_helpers.go b/test/framework/cluster_helpers.go index 1f527b04c172..e80de6ad6bc9 100644 --- a/test/framework/cluster_helpers.go +++ b/test/framework/cluster_helpers.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/controller-runtime/pkg/client" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -40,7 +41,9 @@ type CreateClusterInput struct { // CreateCluster will create the Cluster and InfraCluster objects. func CreateCluster(ctx context.Context, input CreateClusterInput, intervals ...interface{}) { By("creating an InfrastructureCluster resource") - Expect(input.Creator.Create(ctx, input.InfraCluster)).To(Succeed()) + Eventually(func() error { + return input.Creator.Create(ctx, input.InfraCluster) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed()) // This call happens in an eventually because of a race condition with the // webhook server. If the latter isn't fully online then this call will @@ -64,7 +67,9 @@ type GetAllClustersByNamespaceInput struct { // GetAllClustersByNamespace returns the list of Cluster object in a namespace. func GetAllClustersByNamespace(ctx context.Context, input GetAllClustersByNamespaceInput) []*clusterv1.Cluster { clusterList := &clusterv1.ClusterList{} - Expect(input.Lister.List(ctx, clusterList, client.InNamespace(input.Namespace))).To(Succeed(), "Failed to list clusters in namespace %s", input.Namespace) + Eventually(func() error { + return input.Lister.List(ctx, clusterList, client.InNamespace(input.Namespace)) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list clusters in namespace %s", input.Namespace) clusters := make([]*clusterv1.Cluster, len(clusterList.Items)) for i := range clusterList.Items { @@ -184,12 +189,15 @@ func DiscoveryAndWaitForCluster(ctx context.Context, input DiscoveryAndWaitForCl Expect(input.Namespace).ToNot(BeNil(), "Invalid argument. input.Namespace can't be empty when calling DiscoveryAndWaitForCluster") Expect(input.Name).ToNot(BeNil(), "Invalid argument. input.Name can't be empty when calling DiscoveryAndWaitForCluster") - cluster := GetClusterByName(ctx, GetClusterByNameInput{ - Getter: input.Getter, - Name: input.Name, - Namespace: input.Namespace, - }) - Expect(cluster).ToNot(BeNil(), "Failed to get the Cluster object") + var cluster *clusterv1.Cluster + Eventually(func() bool { + cluster = GetClusterByName(ctx, GetClusterByNameInput{ + Getter: input.Getter, + Name: input.Name, + Namespace: input.Namespace, + }) + return cluster != nil + }, retryableOperationTimeout, retryableOperationInterval).Should(BeTrue(), "Failed to get Cluster object %s/%s", input.Namespace, input.Name) // NOTE: We intentionally return the provisioned Cluster because it also contains // the reconciled ControlPlane ref and InfrastructureCluster ref when using a ClusterClass. @@ -226,11 +234,12 @@ func DeleteClusterAndWait(ctx context.Context, input DeleteClusterAndWaitInput, //TODO: consider if to move in another func (what if there are more than one cluster?) log.Logf("Check for all the Cluster API resources being deleted") - resources := GetCAPIResources(ctx, GetCAPIResourcesInput{ - Lister: input.Client, - Namespace: input.Cluster.Namespace, - }) - Expect(resources).To(BeEmpty(), "There are still Cluster API resources in the %q namespace", input.Cluster.Namespace) + Eventually(func() []*unstructured.Unstructured { + return GetCAPIResources(ctx, GetCAPIResourcesInput{ + Lister: input.Client, + Namespace: input.Cluster.Namespace, + }) + }, retryableOperationTimeout, retryableOperationInterval).Should(BeEmpty(), "There are still Cluster API resources in the %q namespace", input.Cluster.Namespace) } // DeleteAllClustersAndWaitInput is the input type for DeleteAllClustersAndWait. diff --git a/test/framework/cluster_proxy.go b/test/framework/cluster_proxy.go index 5b3ccea1c536..efd2996e0b9f 100644 --- a/test/framework/cluster_proxy.go +++ b/test/framework/cluster_proxy.go @@ -248,8 +248,12 @@ func (p *clusterProxy) CollectWorkloadClusterLogs(ctx context.Context, namespace return } - machines, err := getMachinesInCluster(ctx, p.GetClient(), namespace, name) - Expect(err).ToNot(HaveOccurred(), "Failed to get machines for the %s/%s cluster", namespace, name) + var machines *clusterv1.MachineList + Eventually(func() error { + var err error + machines, err = getMachinesInCluster(ctx, p.GetClient(), namespace, name) + return err + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get machines for the %s/%s cluster", namespace, name) for i := range machines.Items { m := &machines.Items[i] @@ -260,8 +264,12 @@ func (p *clusterProxy) CollectWorkloadClusterLogs(ctx context.Context, namespace } } - machinePools, err := getMachinePoolsInCluster(ctx, p.GetClient(), namespace, name) - Expect(err).ToNot(HaveOccurred(), "Failed to get machine pools for the %s/%s cluster", namespace, name) + var machinePools *expv1.MachinePoolList + Eventually(func() error { + var err error + machinePools, err = getMachinePoolsInCluster(ctx, p.GetClient(), namespace, name) + return err + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get machine pools for the %s/%s cluster", namespace, name) for i := range machinePools.Items { mp := &machinePools.Items[i] diff --git a/test/framework/clusterresourceset_helpers.go b/test/framework/clusterresourceset_helpers.go index eb846d10c56a..99f1395644ef 100644 --- a/test/framework/clusterresourceset_helpers.go +++ b/test/framework/clusterresourceset_helpers.go @@ -39,7 +39,9 @@ type GetClusterResourceSetsInput struct { // GetClusterResourceSets returns all ClusterResourceSet objects in a namespace. func GetClusterResourceSets(ctx context.Context, input GetClusterResourceSetsInput) []*addonsv1.ClusterResourceSet { crsList := &addonsv1.ClusterResourceSetList{} - Expect(input.Lister.List(ctx, crsList, client.InNamespace(input.Namespace))).To(Succeed(), "Failed to list ClusterResourceSet objects for namespace %s", input.Namespace) + Eventually(func() error { + return input.Lister.List(ctx, crsList, client.InNamespace(input.Namespace)) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list ClusterResourceSet objects for namespace %s", input.Namespace) clusterResourceSets := make([]*addonsv1.ClusterResourceSet, len(crsList.Items)) for i := range crsList.Items { @@ -78,12 +80,14 @@ func DiscoverClusterResourceSetAndWaitForSuccess(ctx context.Context, input Disc mgmtClient := input.ClusterProxy.GetClient() fmt.Fprintln(GinkgoWriter, "Discovering cluster resource set resources") - clusterResourceSets := GetClusterResourceSets(ctx, GetClusterResourceSetsInput{ - Lister: mgmtClient, - Namespace: input.Cluster.Namespace, - }) - - Expect(clusterResourceSets).NotTo(BeEmpty()) + var clusterResourceSets []*addonsv1.ClusterResourceSet + Eventually(func() bool { + clusterResourceSets = GetClusterResourceSets(ctx, GetClusterResourceSetsInput{ + Lister: mgmtClient, + Namespace: input.Cluster.Namespace, + }) + return len(clusterResourceSets) > 0 + }, retryableOperationTimeout, retryableOperationInterval).Should(BeTrue()) for _, crs := range clusterResourceSets { Expect(crs.Spec.ClusterSelector).NotTo(BeNil()) diff --git a/test/framework/controller_helpers.go b/test/framework/controller_helpers.go index 7c11238c0f87..a4327a37264d 100644 --- a/test/framework/controller_helpers.go +++ b/test/framework/controller_helpers.go @@ -32,7 +32,9 @@ type GetControllerDeploymentsInput struct { // GetControllerDeployments returns all the deployment for the cluster API controllers existing in a management cluster. func GetControllerDeployments(ctx context.Context, input GetControllerDeploymentsInput) []*appsv1.Deployment { deploymentList := &appsv1.DeploymentList{} - Expect(input.Lister.List(ctx, deploymentList, capiProviderOptions()...)).To(Succeed(), "Failed to list deployments for the cluster API controllers") + Eventually(func() error { + return input.Lister.List(ctx, deploymentList, capiProviderOptions()...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list deployments for the cluster API controllers") deployments := make([]*appsv1.Deployment, 0, len(deploymentList.Items)) for i := range deploymentList.Items { diff --git a/test/framework/controlplane_helpers.go b/test/framework/controlplane_helpers.go index 678df36b446e..381c208e8885 100644 --- a/test/framework/controlplane_helpers.go +++ b/test/framework/controlplane_helpers.go @@ -71,7 +71,9 @@ type GetKubeadmControlPlaneByClusterInput struct { // it is necessary to ensure this is already happened before calling it. func GetKubeadmControlPlaneByCluster(ctx context.Context, input GetKubeadmControlPlaneByClusterInput) *controlplanev1.KubeadmControlPlane { controlPlaneList := &controlplanev1.KubeadmControlPlaneList{} - Expect(input.Lister.List(ctx, controlPlaneList, byClusterOptions(input.ClusterName, input.Namespace)...)).To(Succeed(), "Failed to list KubeadmControlPlane object for Cluster %s/%s", input.Namespace, input.ClusterName) + Eventually(func() error { + return input.Lister.List(ctx, controlPlaneList, byClusterOptions(input.ClusterName, input.Namespace)...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list KubeadmControlPlane object for Cluster %s/%s", input.Namespace, input.ClusterName) Expect(len(controlPlaneList.Items)).ToNot(BeNumerically(">", 1), "Cluster %s/%s should not have more than 1 KubeadmControlPlane object", input.Namespace, input.ClusterName) if len(controlPlaneList.Items) == 1 { return &controlPlaneList.Items[0] @@ -205,8 +207,9 @@ func AssertControlPlaneFailureDomains(ctx context.Context, input AssertControlPl } machineList := &clusterv1.MachineList{} - Expect(input.GetLister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption)). - To(Succeed(), "Couldn't list machines for the cluster %q", input.ClusterKey.Name) + Eventually(func() error { + return input.GetLister.List(ctx, machineList, inClustersNamespaceListOption, matchClusterListOption) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Couldn't list machines for the cluster %q", input.ClusterKey.Name) // Count all control plane machine failure domains. for _, machine := range machineList.Items { diff --git a/test/framework/deployment_helpers.go b/test/framework/deployment_helpers.go index 608801ef5bef..0ad731f6b72c 100644 --- a/test/framework/deployment_helpers.go +++ b/test/framework/deployment_helpers.go @@ -36,6 +36,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" utilversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" @@ -185,7 +186,9 @@ func WatchPodMetrics(ctx context.Context, input WatchPodMetricsInput) { Expect(err).NotTo(HaveOccurred(), "Failed to Pods selector for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) pods := &corev1.PodList{} - Expect(input.GetLister.List(ctx, pods, client.InNamespace(input.Deployment.Namespace), client.MatchingLabels(selector))).To(Succeed(), "Failed to list Pods for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) + Eventually(func() error { + return input.GetLister.List(ctx, pods, client.InNamespace(input.Deployment.Namespace), client.MatchingLabels(selector)) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list Pods for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) go func() { defer GinkgoRecover() @@ -324,8 +327,12 @@ func DeployUnevictablePod(ctx context.Context, input DeployUnevictablePodInput) }, } if input.ControlPlane != nil { - serverVersion, err := workloadClient.ServerVersion() - Expect(err).ToNot(HaveOccurred()) + var serverVersion *version.Info + Eventually(func() error { + var err error + serverVersion, err = workloadClient.ServerVersion() + return err + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed()) // Use the control-plane label for Kubernetes version >= v1.20.0. if utilversion.MustParseGeneric(serverVersion.String()).AtLeast(utilversion.MustParseGeneric("v1.20.0")) { @@ -408,7 +415,11 @@ type AddPodDisruptionBudgetInput struct { } func AddPodDisruptionBudget(ctx context.Context, input AddPodDisruptionBudgetInput) { - budget, err := input.ClientSet.PolicyV1beta1().PodDisruptionBudgets(input.Namespace).Create(ctx, input.Budget, metav1.CreateOptions{}) - Expect(budget).NotTo(BeNil()) - Expect(err).To(BeNil(), "podDisruptionBudget needs to be successfully deployed") + Eventually(func() error { + budget, err := input.ClientSet.PolicyV1beta1().PodDisruptionBudgets(input.Namespace).Create(ctx, input.Budget, metav1.CreateOptions{}) + if budget != nil && err == nil { + return nil + } + return fmt.Errorf("podDisruptionBudget needs to be successfully deployed: %v", err) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "podDisruptionBudget needs to be successfully deployed") } diff --git a/test/framework/machine_helpers.go b/test/framework/machine_helpers.go index c91ff11d6d33..ba0ed258f3e1 100644 --- a/test/framework/machine_helpers.go +++ b/test/framework/machine_helpers.go @@ -55,7 +55,9 @@ func GetMachinesByMachineDeployments(ctx context.Context, input GetMachinesByMac opts = append(opts, machineDeploymentOptions(input.MachineDeployment)...) machineList := &clusterv1.MachineList{} - Expect(input.Lister.List(ctx, machineList, opts...)).To(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.Namespace, input.ClusterName) + Eventually(func() error { + return input.Lister.List(ctx, machineList, opts...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.Namespace, input.ClusterName) return machineList.Items } @@ -78,7 +80,9 @@ func GetMachinesByMachineHealthCheck(ctx context.Context, input GetMachinesByMac opts = append(opts, machineHealthCheckOptions(*input.MachineHealthCheck)...) machineList := &clusterv1.MachineList{} - Expect(input.Lister.List(ctx, machineList, opts...)).To(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.MachineHealthCheck.Namespace, input.ClusterName) + Eventually(func() error { + return input.Lister.List(ctx, machineList, opts...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.MachineHealthCheck.Namespace, input.ClusterName) return machineList.Items } @@ -102,7 +106,9 @@ func GetControlPlaneMachinesByCluster(ctx context.Context, input GetControlPlane options := append(byClusterOptions(input.ClusterName, input.Namespace), controlPlaneMachineOptions()...) machineList := &clusterv1.MachineList{} - Expect(input.Lister.List(ctx, machineList, options...)).To(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.Namespace, input.ClusterName) + Eventually(func() error { + return input.Lister.List(ctx, machineList, options...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.Namespace, input.ClusterName) return machineList.Items } diff --git a/test/framework/machinedeployment_helpers.go b/test/framework/machinedeployment_helpers.go index 57d71c3c470d..766251886b82 100644 --- a/test/framework/machinedeployment_helpers.go +++ b/test/framework/machinedeployment_helpers.go @@ -72,7 +72,9 @@ type GetMachineDeploymentsByClusterInput struct { // it is necessary to ensure this is already happened before calling it. func GetMachineDeploymentsByCluster(ctx context.Context, input GetMachineDeploymentsByClusterInput) []*clusterv1.MachineDeployment { deploymentList := &clusterv1.MachineDeploymentList{} - Expect(input.Lister.List(ctx, deploymentList, byClusterOptions(input.ClusterName, input.Namespace)...)).To(Succeed(), "Failed to list MachineDeployments object for Cluster %s/%s", input.Namespace, input.ClusterName) + Eventually(func() error { + return input.Lister.List(ctx, deploymentList, byClusterOptions(input.ClusterName, input.Namespace)...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineDeployments object for Cluster %s/%s", input.Namespace, input.ClusterName) deployments := make([]*clusterv1.MachineDeployment, len(deploymentList.Items)) for i := range deploymentList.Items { @@ -213,9 +215,9 @@ func WaitForMachineDeploymentRollingUpgradeToStart(ctx context.Context, input Wa Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling WaitForMachineDeploymentRollingUpgradeToStarts") log.Logf("Waiting for MachineDeployment rolling upgrade to start") - Eventually(func() bool { + Eventually(func(g Gomega) bool { md := &clusterv1.MachineDeployment{} - Expect(input.Getter.Get(ctx, client.ObjectKey{Namespace: input.MachineDeployment.Namespace, Name: input.MachineDeployment.Name}, md)).To(Succeed()) + g.Expect(input.Getter.Get(ctx, client.ObjectKey{Namespace: input.MachineDeployment.Namespace, Name: input.MachineDeployment.Name}, md)).To(Succeed()) return md.Status.Replicas != md.Status.AvailableReplicas }, intervals...).Should(BeTrue()) } @@ -233,9 +235,9 @@ func WaitForMachineDeploymentRollingUpgradeToComplete(ctx context.Context, input Expect(input.MachineDeployment).ToNot(BeNil(), "Invalid argument. input.MachineDeployment can't be nil when calling WaitForMachineDeploymentRollingUpgradeToComplete") log.Logf("Waiting for MachineDeployment rolling upgrade to complete") - Eventually(func() bool { + Eventually(func(g Gomega) bool { md := &clusterv1.MachineDeployment{} - Expect(input.Getter.Get(ctx, client.ObjectKey{Namespace: input.MachineDeployment.Namespace, Name: input.MachineDeployment.Name}, md)).To(Succeed()) + g.Expect(input.Getter.Get(ctx, client.ObjectKey{Namespace: input.MachineDeployment.Namespace, Name: input.MachineDeployment.Name}, md)).To(Succeed()) return md.Status.Replicas == md.Status.AvailableReplicas }, intervals...).Should(BeTrue()) } diff --git a/test/framework/machinehealthcheck_helpers.go b/test/framework/machinehealthcheck_helpers.go index 314601c41609..24ce2ccef274 100644 --- a/test/framework/machinehealthcheck_helpers.go +++ b/test/framework/machinehealthcheck_helpers.go @@ -101,7 +101,9 @@ type GetMachineHealthChecksForClusterInput struct { // it is necessary to ensure this is already happened before calling it. func GetMachineHealthChecksForCluster(ctx context.Context, input GetMachineHealthChecksForClusterInput) []*clusterv1.MachineHealthCheck { machineHealthCheckList := &clusterv1.MachineHealthCheckList{} - Expect(input.Lister.List(ctx, machineHealthCheckList, byClusterOptions(input.ClusterName, input.Namespace)...)).To(Succeed(), "Failed to list MachineDeployments object for Cluster %s/%s", input.Namespace, input.ClusterName) + Eventually(func() error { + return input.Lister.List(ctx, machineHealthCheckList, byClusterOptions(input.ClusterName, input.Namespace)...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachineHealthCheck object for Cluster %s/%s", input.Namespace, input.ClusterName) machineHealthChecks := make([]*clusterv1.MachineHealthCheck, len(machineHealthCheckList.Items)) for i := range machineHealthCheckList.Items { diff --git a/test/framework/machinepool_helpers.go b/test/framework/machinepool_helpers.go index 1cd34629c9a4..a22d11d1fc78 100644 --- a/test/framework/machinepool_helpers.go +++ b/test/framework/machinepool_helpers.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/gomega" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -51,7 +52,9 @@ func GetMachinePoolsByCluster(ctx context.Context, input GetMachinePoolsByCluste Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling GetMachinePoolsByCluster") mpList := &expv1.MachinePoolList{} - Expect(input.Lister.List(ctx, mpList, byClusterOptions(input.ClusterName, input.Namespace)...)).To(Succeed(), "Failed to list MachinePools object for Cluster %s/%s", input.Namespace, input.ClusterName) + Eventually(func() error { + return input.Lister.List(ctx, mpList, byClusterOptions(input.ClusterName, input.Namespace)...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list MachinePools object for Cluster %s/%s", input.Namespace, input.ClusterName) mps := make([]*expv1.MachinePool, len(mpList.Items)) for i := range mpList.Items { @@ -279,7 +282,13 @@ func getMachinePoolInstanceVersions(ctx context.Context, input GetMachinesPoolIn versions := make([]string, len(instances)) for i, instance := range instances { node := &corev1.Node{} - err := input.WorkloadClusterGetter.Get(ctx, client.ObjectKey{Name: instance.Name}, node) + err := wait.PollImmediate(retryableOperationInterval, retryableOperationTimeout, func() (bool, error) { + err := input.WorkloadClusterGetter.Get(ctx, client.ObjectKey{Name: instance.Name}, node) + if err != nil { + return false, nil //nolint:nilerr + } + return true, nil + }) if err != nil { versions[i] = "unknown" } else { diff --git a/test/framework/machines.go b/test/framework/machines.go index a811dcb6cef5..4cf6854bed6a 100644 --- a/test/framework/machines.go +++ b/test/framework/machines.go @@ -39,7 +39,9 @@ func WaitForClusterMachineNodeRefs(ctx context.Context, input WaitForClusterMach By("Waiting for the machines' nodes to exist") machines := &clusterv1.MachineList{} - Expect(input.GetLister.List(ctx, machines, byClusterOptions(input.Cluster.Name, input.Cluster.Namespace)...)).To(Succeed(), "Failed to get Cluster machines %s/%s", input.Cluster.Namespace, input.Cluster.Name) + Eventually(func() error { + return input.GetLister.List(ctx, machines, byClusterOptions(input.Cluster.Name, input.Cluster.Namespace)...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get Cluster machines %s/%s", input.Cluster.Namespace, input.Cluster.Name) Eventually(func() (count int, err error) { for _, m := range machines.Items { machine := &clusterv1.Machine{} @@ -65,7 +67,9 @@ func WaitForClusterMachinesReady(ctx context.Context, input WaitForClusterMachin By("Waiting for the machines' nodes to be ready") machines := &clusterv1.MachineList{} - Expect(input.GetLister.List(ctx, machines, byClusterOptions(input.Cluster.Name, input.Cluster.Namespace)...)).To(Succeed(), "Failed to get Cluster machines %s/%s", input.Cluster.Namespace, input.Cluster.Name) + Eventually(func() error { + return input.GetLister.List(ctx, machines, byClusterOptions(input.Cluster.Name, input.Cluster.Namespace)...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get Cluster machines %s/%s", input.Cluster.Namespace, input.Cluster.Name) Eventually(func() (count int, err error) { for _, m := range machines.Items { machine := &clusterv1.Machine{}