Skip to content

Commit

Permalink
Merge pull request #9582 from k8s-infra-cherrypick-robot/cherry-pick-…
Browse files Browse the repository at this point in the history
…9570-to-release-1.5

[release-1.5] 🐛 Retry Node delete when CCT is locked
  • Loading branch information
k8s-ci-robot authored Oct 20, 2023
2 parents 2997928 + 789b3a0 commit bd94113
Showing 1 changed file with 9 additions and 2 deletions.
11 changes: 9 additions & 2 deletions internal/controllers/machine/machine_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -604,7 +604,11 @@ func (r *Reconciler) drainNode(ctx context.Context, cluster *clusterv1.Cluster,

restConfig, err := r.Tracker.GetRESTConfig(ctx, util.ObjectKey(cluster))
if err != nil {
log.Error(err, "Error creating a remote client while deleting Machine, won't retry")
if errors.Is(err, remote.ErrClusterLocked) {
log.V(5).Info("Requeuing drain Node because another worker has the lock on the ClusterCacheTracker")
return ctrl.Result{Requeue: true}, nil
}
log.Error(err, "Error creating a remote client for cluster while draining Node, won't retry")
return ctrl.Result{}, nil
}
restConfig = rest.CopyConfig(restConfig)
Expand Down Expand Up @@ -700,7 +704,10 @@ func (r *Reconciler) deleteNode(ctx context.Context, cluster *clusterv1.Cluster,

remoteClient, err := r.Tracker.GetClient(ctx, util.ObjectKey(cluster))
if err != nil {
log.Error(err, "Error creating a remote client for cluster while deleting Machine, won't retry")
if errors.Is(err, remote.ErrClusterLocked) {
return errors.Wrapf(err, "failed deleting Node because another worker has the lock on the ClusterCacheTracker")
}
log.Error(err, "Error creating a remote client for cluster while deleting Node, won't retry")
return nil
}

Expand Down

0 comments on commit bd94113

Please sign in to comment.