From fad64b3ea8051125ff5dc8c3a1bd6c090514dc80 Mon Sep 17 00:00:00 2001
From: Cameron McAvoy <cmcavoy@indeed.com>
Date: Fri, 23 Jun 2023 14:23:52 -0500
Subject: [PATCH] Allow configuring the kubernetes rest client timeout for
 draining nodes

---
 controllers/alias.go                               | 5 +++++
 internal/controllers/machine/machine_controller.go | 6 ++++++
 main.go                                            | 5 +++++
 3 files changed, 16 insertions(+)

diff --git a/controllers/alias.go b/controllers/alias.go
index 2e6af641de4e..7ac1a6b30b95 100644
--- a/controllers/alias.go
+++ b/controllers/alias.go
@@ -18,6 +18,7 @@ package controllers
 
 import (
 	"context"
+	"time"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -64,6 +65,9 @@ type MachineReconciler struct {
 
 	// WatchFilterValue is the label value used to filter events prior to reconciliation.
 	WatchFilterValue string
+
+	// DrainKubeTimeout is the maximum length of time to wait for a response before giving up on kubernetes to drain a node.
+	DrainKubeTimeout time.Duration
 }
 
 func (r *MachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
@@ -72,6 +76,7 @@ func (r *MachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manag
 		APIReader:        r.APIReader,
 		Tracker:          r.Tracker,
 		WatchFilterValue: r.WatchFilterValue,
+		DrainKubeTimeout: r.DrainKubeTimeout,
 	}).SetupWithManager(ctx, mgr, options)
 }
 
diff --git a/internal/controllers/machine/machine_controller.go b/internal/controllers/machine/machine_controller.go
index f486fb5586e7..af26166e103b 100644
--- a/internal/controllers/machine/machine_controller.go
+++ b/internal/controllers/machine/machine_controller.go
@@ -30,6 +30,7 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	kubedrain "k8s.io/kubectl/pkg/drain"
@@ -80,6 +81,9 @@ type Reconciler struct {
 	// WatchFilterValue is the label value used to filter events prior to reconciliation.
 	WatchFilterValue string
 
+	// DrainKubeTimeout is the maximum length of time to wait for a response before giving up on kubernetes to drain a node.
+	DrainKubeTimeout time.Duration
+
 	controller      controller.Controller
 	recorder        record.EventRecorder
 	externalTracker external.ObjectTracker
@@ -601,6 +605,8 @@ func (r *Reconciler) drainNode(ctx context.Context, cluster *clusterv1.Cluster,
 		log.Error(err, "Error creating a remote client while deleting Machine, won't retry")
 		return ctrl.Result{}, nil
 	}
+	restConfig = rest.CopyConfig(restConfig)
+	restConfig.Timeout = r.DrainKubeTimeout
 	kubeClient, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
 		log.Error(err, "Error creating a remote client while deleting Machine, won't retry")
diff --git a/main.go b/main.go
index 5061894a57c0..98d0f3acd3e8 100644
--- a/main.go
+++ b/main.go
@@ -99,6 +99,7 @@ var (
 	syncPeriod                    time.Duration
 	restConfigQPS                 float32
 	restConfigBurst               int
+	drainKubeTimeout              time.Duration
 	webhookPort                   int
 	webhookCertDir                string
 	healthAddr                    string
@@ -198,6 +199,9 @@ func InitFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&restConfigBurst, "kube-api-burst", 30,
 		"Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default 30")
 
+	fs.DurationVar(&drainKubeTimeout, "drain-api-timeout", time.Second*10,
+		"The maximum length of time to wait for a response before giving up on kubernetes to drain a node. A value of zero means no timeout.")
+
 	fs.IntVar(&webhookPort, "webhook-port", 9443,
 		"Webhook Server port")
 
@@ -229,6 +233,7 @@ func main() {
 	restConfig := ctrl.GetConfigOrDie()
 	restConfig.QPS = restConfigQPS
 	restConfig.Burst = restConfigBurst
+	restConfig.Timeout = drainKubeTimeout
 	restConfig.UserAgent = remote.DefaultClusterAPIUserAgent(controllerName)
 
 	minVer := version.MinimumKubernetesVersion