From 51325d5737a44286716efeaaf61db792cddfd864 Mon Sep 17 00:00:00 2001 From: Tahsin Date: Thu, 26 Sep 2019 23:33:43 +0600 Subject: [PATCH] Machine deletion: try up to 10s to delete the Node, then move on --- controllers/machine_controller.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/controllers/machine_controller.go b/controllers/machine_controller.go index 64edb42c983a..bef6dd9007cb 100644 --- a/controllers/machine_controller.go +++ b/controllers/machine_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "sync" + "time" "github.com/go-logr/logr" "github.com/pkg/errors" @@ -27,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/record" "k8s.io/klog" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha2" @@ -183,9 +185,15 @@ func (r *MachineReconciler) reconcileDelete(ctx context.Context, cluster *cluste } } else { klog.Infof("Deleting node %q for machine %q", m.Status.NodeRef.Name, m.Name) - if err := r.deleteNode(ctx, cluster, m.Status.NodeRef.Name); err != nil && !apierrors.IsNotFound(err) { - klog.Errorf("Error deleting node %q for machine %q: %v", m.Status.NodeRef.Name, m.Name, err) - return ctrl.Result{}, err + waitErr := wait.PollImmediate(2*time.Second, 10*time.Second, func() (bool, error) { + if err = r.deleteNode(ctx, cluster, m.Status.NodeRef.Name); err != nil && !apierrors.IsNotFound(err) { + return false, nil + } + return true, nil + }) + if waitErr != nil { + r.Log.Error(err, "timed out deleting Machine's node, moving on", "node", m.Status.NodeRef.Name) + r.recorder.Eventf(m, corev1.EventTypeWarning, "FailedDeleteNode", "error deleting Machine's node: %v", err) } }