chore: retry with client-go native routine

on2itsecurity · Apr 22, 2022 · c2fdf81 · c2fdf81
1 parent f6db366
commit c2fdf81
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 17 deletions.
diff --git a/pkg/cluster/tolerated_failure.go b/pkg/cluster/tolerated_failure.go
@@ -28,7 +28,7 @@ func (c *Cluster) updatePodDisruptionBudget(ctx context.Context) {
 		if c.cluster.Spec.PodDisruptionBudget {
 			c.logger.Infof("Updating disruption budget, amount of pods needed for a quorum changed from %d to %d", c.status.MinAvailable, currentlyMinAvailable)
 
-			err := k8sutil.UpdateOrCreatePodDisruptionBudget(ctx, c.config.KubeCli, c.cluster.Namespace, c.cluster.Name, currentlyMinAvailable, c.cluster.AsOwner(), true)
+			err := k8sutil.UpdateOrCreatePodDisruptionBudget(ctx, c.config.KubeCli, c.cluster.Namespace, c.cluster.Name, currentlyMinAvailable, c.cluster.AsOwner())
 			if err != nil {
 				c.logger.Errorf("failed to create/update pod disruption budget: %v", err)
 				return

diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go
@@ -40,6 +40,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // for gcp auth
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/util/retry"
 )
 
 const (
@@ -550,26 +551,26 @@ func PatchDeployment(ctx context.Context, kubecli kubernetes.Interface, namespac
 	return err
 }
 
-func UpdateOrCreatePodDisruptionBudget(ctx context.Context, kubecli kubernetes.Interface, namespace, name string, minAvailable int, owner metav1.OwnerReference, retryOptimisticLockingUpdate bool) error {
+func UpdateOrCreatePodDisruptionBudget(ctx context.Context, kubecli kubernetes.Interface, namespace, name string, minAvailable int, owner metav1.OwnerReference) error {
 	pdb := newEtcdPodDisruptionBudgetManifest(name, name, &intstr.IntOrString{IntVal: int32(minAvailable)})
-
 	addOwnerRefToObject(pdb.GetObjectMeta(), owner)
 
-	od, err := kubecli.PolicyV1beta1().PodDisruptionBudgets(namespace).Get(ctx, name, metav1.GetOptions{})
-	if apierrors.IsNotFound(err) {
-		_, err := kubecli.PolicyV1beta1().PodDisruptionBudgets(namespace).Create(ctx, pdb, metav1.CreateOptions{})
-		return err
-	}
-	pdb.Status = od.Status
-	pdb.ObjectMeta.ResourceVersion = od.ObjectMeta.ResourceVersion // Optimistic locking
-	_, err = kubecli.PolicyV1beta1().PodDisruptionBudgets(namespace).Update(ctx, pdb, metav1.UpdateOptions{})
-	if apierrors.IsConflict(err) && retryOptimisticLockingUpdate {
-		return UpdateOrCreatePodDisruptionBudget(ctx, kubecli, namespace, name, minAvailable, owner, false)
-	}
-	if err != nil {
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		od, err := kubecli.PolicyV1beta1().PodDisruptionBudgets(namespace).Get(ctx, name, metav1.GetOptions{})
+		if IsKubernetesResourceNotFoundError(err) {
+			_, err := kubecli.PolicyV1beta1().PodDisruptionBudgets(namespace).Create(ctx, pdb, metav1.CreateOptions{})
+			return err
+		}
+		if err != nil {
+			return err
+		}
+
+		pdb.Status = od.Status
+		pdb.ObjectMeta.ResourceVersion = od.ObjectMeta.ResourceVersion // Optimistic locking
+		_, err = kubecli.PolicyV1beta1().PodDisruptionBudgets(namespace).Update(ctx, pdb, metav1.UpdateOptions{})
+
 		return err
-	}
-	return nil
+	})
 }
 
 func CascadeDeleteOptions(gracePeriodSeconds int64) *metav1.DeleteOptions {