fix: for rollouts getting stuck due to bad rs informer updates

Signed-off-by: Zach Aller <[email protected]>
argoproj · Nov 30, 2023 · 6e610a4 · 6e610a4
1 parent 8cae284
commit 6e610a4
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 8 deletions.
diff --git a/rollout/replicaset.go b/rollout/replicaset.go
@@ -37,7 +37,11 @@ func (c *rolloutContext) removeScaleDownDelay(rs *appsv1.ReplicaSet) error {
 	_, err := c.kubeclientset.AppsV1().ReplicaSets(rs.Namespace).Patch(ctx, rs.Name, patchtypes.JSONPatchType, []byte(patch), metav1.PatchOptions{})
 	if err == nil {
 		c.log.Infof("Removed '%s' annotation from RS '%s'", v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey, rs.Name)
-		c.replicaSetInformer.GetIndexer().Update(rs)
+		err = c.replicaSetInformer.GetIndexer().Update(rs)
+		if err != nil {
+			return fmt.Errorf("error updating replicaset informer in removeScaleDownDelay", err)
+		}
+
 	}
 	return err
 }
@@ -62,7 +66,10 @@ func (c *rolloutContext) addScaleDownDelay(rs *appsv1.ReplicaSet, scaleDownDelay
 	rs, err := c.kubeclientset.AppsV1().ReplicaSets(rs.Namespace).Patch(ctx, rs.Name, patchtypes.JSONPatchType, []byte(patch), metav1.PatchOptions{})
 	if err == nil {
 		c.log.Infof("Set '%s' annotation on '%s' to %s (%s)", v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey, rs.Name, deadline, scaleDownDelaySeconds)
-		c.replicaSetInformer.GetIndexer().Update(rs)
+		err = c.replicaSetInformer.GetIndexer().Update(rs)
+		if err != nil {
+			return fmt.Errorf("error updating replicaset informer in addScaleDownDelay", err)
+		}
 	}
 	return err
 }

diff --git a/rollout/sync.go b/rollout/sync.go
@@ -91,7 +91,10 @@ func (c *rolloutContext) syncReplicaSetRevision() (*appsv1.ReplicaSet, error) {
 			return nil, fmt.Errorf("error updating replicaset revision: %v", err)
 		}
 		c.log.Infof("Synced revision on ReplicaSet '%s' to '%s'", rs.Name, newRevision)
-		c.replicaSetInformer.GetIndexer().Update(rs)
+		err = c.replicaSetInformer.GetIndexer().Update(rs)
+		if err != nil {
+			return nil, fmt.Errorf("error updating replicaset informer in syncReplicaSetRevision: %v", err)
+		}
 		return rs, nil
 	}
 
@@ -373,11 +376,17 @@ func (c *rolloutContext) scaleReplicaSet(rs *appsv1.ReplicaSet, newScale int32,
 			delete(rsCopy.Annotations, v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey)
 		}
 		rs, err = c.kubeclientset.AppsV1().ReplicaSets(rsCopy.Namespace).Update(ctx, rsCopy, metav1.UpdateOptions{})
-		if err == nil && sizeNeedsUpdate {
-			scaled = true
-			revision, _ := replicasetutil.Revision(rs)
-			c.recorder.Eventf(rollout, record.EventOptions{EventReason: conditions.ScalingReplicaSetReason}, conditions.ScalingReplicaSetMessage, scalingOperation, rs.Name, revision, oldScale, newScale)
-			c.replicaSetInformer.GetIndexer().Update(rs)
+		if err == nil {
+			err = c.replicaSetInformer.GetIndexer().Update(rs)
+			if err != nil {
+				err = fmt.Errorf("error updating replicaset informer in scaleReplicaSet", c.replicaSetInformer.GetIndexer().Update(rs))
+				return scaled, rs, err
+			}
+			if sizeNeedsUpdate {
+				scaled = true
+				revision, _ := replicasetutil.Revision(rs)
+				c.recorder.Eventf(rollout, record.EventOptions{EventReason: conditions.ScalingReplicaSetReason}, conditions.ScalingReplicaSetMessage, scalingOperation, rs.Name, revision, oldScale, newScale)
+			}
 		}
 	}
 	return scaled, rs, err