Add tests for remediation logic

controllers/machinehealthcheck_targets_test.go

@@ -17,13 +17,18 @@ limitations under the License.
 package controllers
 
 import (
+	"context"
+	"fmt"
 	"testing"
 	"time"
 
 	. "github.com/onsi/gomega"
+	gtypes "github.com/onsi/gomega/types"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/record"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
@@ -327,7 +332,274 @@ func TestHealthCheckTargets(t *testing.T) {
 	}
 }
 
+func TestRemediate(t *testing.T) {
+	namespace := "test-mhc"
+	clusterName := "test-cluster"
+	labels := map[string]string{"cluster": clusterName, "machine-group": "foo"}
+
+	machineSetORs := []metav1.OwnerReference{{Kind: "MachineSet"}}
+
+	workerNode := newTestNode("worker-node")
+	workerMachine := newTestMachine("worker-machine", namespace, clusterName, workerNode.Name, labels)
+	workerMachine.SetOwnerReferences(machineSetORs)
+	workerMachineUnowned := newTestMachine("worker-machine", namespace, clusterName, workerNode.Name, labels)
+
+	masterNode := newTestNode("master-node")
+	if masterNode.Labels == nil {
+		masterNode.Labels = make(map[string]string)
+	}
+	masterNode.Labels[nodeMasterLabel] = ""
+
+	masterMachine := newTestMachine("master-machine", namespace, clusterName, masterNode.Name, labels)
+	masterMachine.SetOwnerReferences(machineSetORs)
+	if masterMachine.Labels == nil {
+		masterMachine.Labels = make(map[string]string)
+	}
+	masterMachine.Labels[machineControlPlaneLabel] = ""
+
+	testCases := []struct {
+		name          string
+		node          *corev1.Node
+		machine       *clusterv1.Machine
+		expectErr     bool
+		expectDeleted bool
+		expectEvents  []string
+	}{
+		{
+			name:          "when the machine is not owned by a machineset",
+			node:          workerNode,
+			machine:       workerMachineUnowned,
+			expectErr:     false,
+			expectDeleted: false,
+			expectEvents:  []string{},
+		},
+		{
+			name:          "when the node is a worker with a machine owned by a machineset",
+			node:          workerNode,
+			machine:       workerMachine,
+			expectErr:     false,
+			expectDeleted: true,
+			expectEvents:  []string{EventMachineDeleted},
+		},
+		{
+			name:          "when the node is a master",
+			node:          masterNode,
+			machine:       masterMachine,
+			expectErr:     false,
+			expectDeleted: false,
+			expectEvents:  []string{EventSkippedMaster},
+		},
+		{
+			name:          "when the machine is a control-plane machine",
+			node:          nil,
+			machine:       masterMachine,
+			expectErr:     false,
+			expectDeleted: false,
+			expectEvents:  []string{EventSkippedMaster},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			gs := NewGomegaWithT(t)
+
+			target := &healthCheckTarget{
+				Node:    tc.node,
+				Machine: tc.machine,
+				MHC:     newTestMachineHealthCheck("mhc", namespace, clusterName, labels),
+			}
+
+			fakeRecorder := record.NewFakeRecorder(2)
+			gs.Expect(clusterv1.AddToScheme(scheme.Scheme)).To(Succeed())
+			k8sClient := fake.NewFakeClientWithScheme(scheme.Scheme, target.Machine)
+			if target.Node != nil {
+				gs.Expect(k8sClient.Create(context.TODO(), target.Node)).To(Succeed())
+			}
+
+			// Run rememdiation
+			err := target.remediate(log.Log, k8sClient, fakeRecorder)
+			gs.Expect(err != nil).To(Equal(tc.expectErr))
+
+			machine := &clusterv1.Machine{}
+			key := types.NamespacedName{Namespace: tc.machine.Namespace, Name: tc.machine.Name}
+			err = k8sClient.Get(context.TODO(), key, machine)
+
+			// Check if the machine was deleted or not
+			if tc.expectDeleted {
+				gs.Expect(errors.IsNotFound(err)).To(BeTrue())
+			} else {
+				gs.Expect(err).ToNot(HaveOccurred())
+				gs.Expect(machine).To(Equal(target.Machine))
+			}
+
+			// Check which event types were sent
+			gs.Expect(fakeRecorder.Events).To(HaveLen(len(tc.expectEvents)))
+			receivedEvents := []string{}
+			eventMatchers := []gtypes.GomegaMatcher{}
+			for _, ev := range tc.expectEvents {
+				receivedEvents = append(receivedEvents, <-fakeRecorder.Events)
+				eventMatchers = append(eventMatchers, ContainSubstring(fmt.Sprintf(" %s ", ev)))
+			}
+			gs.Expect(receivedEvents).To(ConsistOf(eventMatchers))
+		})
+	}
+}
+
+func TestHasMachineSetOwner(t *testing.T) {
+	machineSetOR := metav1.OwnerReference{
+		Kind: "MachineSet",
+	}
+
+	machineDeploymentOR := metav1.OwnerReference{
+		Kind: "MachineDeployment",
+	}
+
+	testCases := []struct {
+		name            string
+		ownerReferences []metav1.OwnerReference
+		hasOwner        bool
+	}{
+		{
+			name:            "with no owner references",
+			ownerReferences: []metav1.OwnerReference{},
+			hasOwner:        false,
+		},
+		{
+			name:            "with a MachineDeployment owner reference",
+			ownerReferences: []metav1.OwnerReference{machineDeploymentOR},
+			hasOwner:        false,
+		},
+		{
+			name:            "with a MachineSet owner reference",
+			ownerReferences: []metav1.OwnerReference{machineSetOR},
+			hasOwner:        true,
+		},
+		{
+			name:            "with a MachineSet and MachineDeployment owner reference",
+			ownerReferences: []metav1.OwnerReference{machineSetOR, machineDeploymentOR},
+			hasOwner:        true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			gs := NewGomegaWithT(t)
+
+			machine := newTestMachine("machine", "test-mhc", "test-cluster", "node", map[string]string{})
+			machine.SetOwnerReferences(tc.ownerReferences)
+
+			target := &healthCheckTarget{
+				Machine: machine,
+			}
+
+			gs.Expect(target.hasMachineSetOwner()).To(Equal(tc.hasOwner))
+		})
+	}
+}
+
+func TestIsMaster(t *testing.T) {
+	namespace := "test-mhc"
+	clusterName := "test-cluster"
+	labels := map[string]string{"cluster": clusterName, "machine-group": "foo"}
+
+	workerNode := newTestNode("worker-node")
+	workerMachine := newTestMachine("worker-machine", namespace, clusterName, workerNode.Name, labels)
+
+	masterNode1 := newTestNode("master-node")
+	if masterNode1.Labels == nil {
+		masterNode1.Labels = make(map[string]string)
+	}
+	masterNode1.Labels[nodeMasterLabel] = ""
+
+	masterMachine1 := newTestMachine("master-machine", namespace, clusterName, masterNode1.Name, labels)
+	if masterMachine1.Labels == nil {
+		masterMachine1.Labels = make(map[string]string)
+	}
+	masterMachine1.Labels[machineControlPlaneLabel] = ""
+
+	masterNode2 := newTestNode("master-node")
+	if masterNode2.Labels == nil {
+		masterNode2.Labels = make(map[string]string)
+	}
+	masterNode2.Labels[nodeMasterLabel] = "abcdef"
+
+	masterMachine2 := newTestMachine("master-machine", namespace, clusterName, masterNode2.Name, labels)
+	if masterMachine2.Labels == nil {
+		masterMachine2.Labels = make(map[string]string)
+	}
+	masterMachine2.Labels[machineControlPlaneLabel] = "abcdef"
+
+	testCases := []struct {
+		name     string
+		node     *corev1.Node
+		machine  *clusterv1.Machine
+		isMaster bool
+	}{
+		{
+			name:     "when the node and machine are nil",
+			node:     nil,
+			machine:  nil,
+			isMaster: false,
+		},
+		{
+			name:     "when the node is not a control-plane node",
+			node:     workerNode,
+			machine:  workerMachine,
+			isMaster: false,
+		},
+		{
+			name:     "when the node is missing and the machine is not a control-plane machine",
+			node:     nil,
+			machine:  workerMachine,
+			isMaster: false,
+		},
+		{
+			name:     "when the node is a control-plane node",
+			node:     masterNode1,
+			machine:  masterMachine1,
+			isMaster: true,
+		},
+		{
+			name:     "when the node is missing and the machine is a control-plane machine",
+			node:     nil,
+			machine:  masterMachine1,
+			isMaster: true,
+		},
+		{
+			name:     "when the node master label has a value",
+			node:     masterNode2,
+			machine:  masterMachine2,
+			isMaster: true,
+		},
+		{
+			name:     "when the machine control-plane label has a value",
+			node:     nil,
+			machine:  masterMachine2,
+			isMaster: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			gs := NewGomegaWithT(t)
+
+			target := &healthCheckTarget{
+				Node:    tc.node,
+				Machine: tc.machine,
+			}
+
+			gs.Expect(target.isMaster()).To(Equal(tc.isMaster))
+		})
+	}
+}
+
 func newTestMachine(name, namespace, clusterName, nodeName string, labels map[string]string) *clusterv1.Machine {
+	// Copy the labels so that the map is unique to each test Machine
+	l := make(map[string]string)
+	for k, v := range labels {
+		l[k] = v
+	}
+
 	bootstrap := "bootstrap"
 	return &clusterv1.Machine{
 		TypeMeta: metav1.TypeMeta{
@@ -337,7 +609,7 @@ func newTestMachine(name, namespace, clusterName, nodeName string, labels map[st
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: namespace,
-			Labels:    labels,
+			Labels:    l,
 		},
 		Spec: clusterv1.MachineSpec{
 			ClusterName: clusterName,