Move node draining from actuator into machine controller

Node draining is a generic operation independent of a specific actuator.
Thus, it makes sense to move the code from actuator into the machine controllers.
The node draining code itself is imported from github.com/openshift/kubernetes-drain.

At the same time it's currently impossible to use the controller-runtime client for node draining
due to missing Patch operation (kubernetes-sigs/controller-runtime#235).
Thus, the machine controller needs to initialize kubeclient as well in order to
implement the node draining logic. Once the Patch operation is implemented,
the draining logic can be updated to replace kube client with controller runtime client.

Also, initialize event recorder to generate node draining event.

pkg/actuators/machine/actuator.go

@@ -21,14 +21,12 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/go-log/log/info"
 	"github.com/golang/glog"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	errorutil "k8s.io/apimachinery/pkg/util/errors"
-	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
 
@@ -41,8 +39,6 @@ import (
 
 	awsclient "sigs.k8s.io/cluster-api-provider-aws/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	kubedrain "github.com/openshift/kubernetes-drain"
 )
 
 const (
@@ -55,8 +51,6 @@ const (
 
 	// MachineCreationFailed indicates that machine creation failed
 	MachineCreationFailed = "MachineCreationFailed"
-	// ExcludeNodeDrainingAnnotation annotation explicitly skips node draining if set
-	ExcludeNodeDrainingAnnotation = "machine.openshift.io/exclude-node-draining"
 )
 
 // Actuator is the AWS-specific actuator for the Cluster API machine controller
@@ -255,50 +249,6 @@ func (gl *glogLogger) Logf(format string, v ...interface{}) {
 
 // DeleteMachine deletes an AWS instance
 func (a *Actuator) DeleteMachine(cluster *machinev1.Cluster, machine *machinev1.Machine) error {
-	// Drain node before deleting
-	// If a machine is not linked to a node, just delete the machine. Since a node
-	// can be unlinked from a machine when the node goes NotReady and is removed
-	// by cloud controller manager. In that case some machines would never get
-	// deleted without a manual intervention.
-	if _, exists := machine.ObjectMeta.Annotations[ExcludeNodeDrainingAnnotation]; !exists && machine.Status.NodeRef != nil {
-		glog.Infof("Draining node before delete")
-		if a.config == nil {
-			err := fmt.Errorf("missing client config, unable to build kube client")
-			glog.Error(err)
-			return err
-		}
-		kubeClient, err := kubernetes.NewForConfig(a.config)
-		if err != nil {
-			return fmt.Errorf("unable to build kube client: %v", err)
-		}
-		node, err := kubeClient.CoreV1().Nodes().Get(machine.Status.NodeRef.Name, metav1.GetOptions{})
-		if err != nil {
-			return fmt.Errorf("unable to get node %q: %v", machine.Status.NodeRef.Name, err)
-		}
-
-		if err := kubedrain.Drain(
-			kubeClient,
-			[]*corev1.Node{node},
-			&kubedrain.DrainOptions{
-				Force:              true,
-				IgnoreDaemonsets:   true,
-				DeleteLocalData:    true,
-				GracePeriodSeconds: -1,
-				Logger:             info.New(glog.V(0)),
-				// If a pod is not evicted in 20 second, retry the eviction next time the
-				// machine gets reconciled again (to allow other machines to be reconciled)
-				Timeout: 20 * time.Second,
-			},
-		); err != nil {
-			// Machine still tries to terminate after drain failure
-			glog.Warningf("drain failed for machine %q: %v", machine.Name, err)
-			return &clustererror.RequeueAfterError{RequeueAfter: requeueAfterSeconds * time.Second}
-		}
-
-		glog.Infof("drain successful for machine %q", machine.Name)
-		a.eventRecorder.Eventf(machine, corev1.EventTypeNormal, "Deleted", "Node %q drained", node.Name)
-	}
-
 	machineProviderConfig, err := providerConfigFromMachine(a.client, machine, a.codec)
 	if err != nil {
 		return a.handleMachineError(machine, apierrors.InvalidMachineConfiguration("error decoding MachineProviderConfig: %v", err), deleteEventAction)

pkg/actuators/machine/stubs.go

@@ -13,6 +13,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	machinev1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1"
+	machinecontroller "github.com/openshift/cluster-api/pkg/controller/machine"
 	providerconfigv1 "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsproviderconfig/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-aws/test/utils"
 )
@@ -119,7 +120,7 @@ func stubMachine() (*machinev1.Machine, error) {
 			},
 			Annotations: map[string]string{
 				// skip node draining since it's not mocked
-				ExcludeNodeDrainingAnnotation: "",
+				machinecontroller.ExcludeNodeDrainingAnnotation: "",
 			},
 		},
 

test/machines/machines_test.go

@@ -18,6 +18,7 @@ import (
 	"sigs.k8s.io/cluster-api-provider-aws/test/utils"
 
 	MachineV1beta1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1"
+	machinecontroller "github.com/openshift/cluster-api/pkg/controller/machine"
 	apiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -136,7 +137,7 @@ var _ = framework.SigKubeDescribe("Machines", func() {
 			if testMachine.Annotations == nil {
 				testMachine.Annotations = map[string]string{}
 			}
-			testMachine.Annotations[machineutils.ExcludeNodeDrainingAnnotation] = ""
+			testMachine.Annotations[machinecontroller.ExcludeNodeDrainingAnnotation] = ""
 			f.CreateMachineAndWait(testMachine, acw)
 			machinesToDelete.AddMachine(testMachine, f, acw)
 
@@ -214,7 +215,7 @@ var _ = framework.SigKubeDescribe("Machines", func() {
 			if testMachine.Annotations == nil {
 				testMachine.Annotations = map[string]string{}
 			}
-			testMachine.Annotations[machineutils.ExcludeNodeDrainingAnnotation] = ""
+			testMachine.Annotations[machinecontroller.ExcludeNodeDrainingAnnotation] = ""
 			f.CreateMachineAndWait(testMachine, acw)
 			machinesToDelete.AddMachine(testMachine, f, acw)
 
@@ -278,7 +279,7 @@ var _ = framework.SigKubeDescribe("Machines", func() {
 			if masterMachine.Annotations == nil {
 				masterMachine.Annotations = map[string]string{}
 			}
-			masterMachine.Annotations[machineutils.ExcludeNodeDrainingAnnotation] = ""
+			masterMachine.Annotations[machinecontroller.ExcludeNodeDrainingAnnotation] = ""
 			f.CreateMachineAndWait(masterMachine, acw)
 			machinesToDelete.AddMachine(masterMachine, f, acw)
 
@@ -326,7 +327,7 @@ var _ = framework.SigKubeDescribe("Machines", func() {
 			if workerMachineSet.Annotations == nil {
 				workerMachineSet.Annotations = map[string]string{}
 			}
-			workerMachineSet.Annotations[machineutils.ExcludeNodeDrainingAnnotation] = ""
+			workerMachineSet.Annotations[machinecontroller.ExcludeNodeDrainingAnnotation] = ""
 			fmt.Printf("workerMachineSet: %#v\n", workerMachineSet)
 			clusterFramework.CreateMachineSetAndWait(workerMachineSet, acw)
 			machinesToDelete.AddMachineSet(workerMachineSet, clusterFramework, acw)