Add nfd E2E tests for tainting feature

Extend current E2E tests to check tainting feature of nfd implemented
in kubernetes-sigs#910

Signed-off-by: Feruzjon Muyassarov <[email protected]>

test/e2e/data/nodefeaturerule-3-updated.yaml

@@ -0,0 +1,32 @@
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeatureRule
+metadata:
+  name: e2e-test-3
+spec:
+  rules:
+    # Positive test expected to set the taints
+    - name: "e2e-taint-test-1"
+      taints:
+        - effect: PreferNoSchedule
+          key: "nfd.node.kubernetes.io/fake-special-node"
+          value: "exists"
+        - effect: NoExecute
+          key: "nfd.node.kubernetes.io/foo"
+          value: "exists"
+      matchFeatures:
+        - feature: "fake.attribute"
+          matchExpressions:
+            "attr_1": {op: IsTrue}
+            "attr_2": {op: IsFalse}
+
+    # Negative test not supposed to set the taints
+    - name: "e2e-taint-test-2"
+      taints:
+        - effect: PreferNoSchedule
+          key: "nfd.node.kubernetes.io/fake-cpu"
+          value: "true"
+      matchFeatures:
+        - feature: "fake.attribute"
+          matchExpressions:
+            "attr_1": {op: IsTrue}
+            "attr_2": {op: IsTrue}

test/e2e/data/nodefeaturerule-3.yaml

@@ -0,0 +1,35 @@
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeatureRule
+metadata:
+  name: e2e-test-3
+spec:
+  rules:
+    # Positive test expected to set the taints
+    - name: "e2e-taint-test-1"
+      taints:
+        - effect: PreferNoSchedule
+          key: "nfd.node.kubernetes.io/fake-special-node"
+          value: "exists"
+        - effect: NoExecute
+          key: "nfd.node.kubernetes.io/fake-dedicated-node"
+          value: "true"
+        - effect: "NoExecute"
+          key: "nfd.node.kubernetes.io/performance-optimized-node"
+          value: "true"
+      matchFeatures:
+        - feature: "fake.attribute"
+          matchExpressions:
+            "attr_1": {op: IsTrue}
+            "attr_2": {op: IsFalse}
+
+    # Negative test not supposed to set the taints
+    - name: "e2e-taint-test-2"
+      taints:
+        - effect: PreferNoSchedule
+          key: "nfd.node.kubernetes.io/fake-special-cpu"
+          value: "true"
+      matchFeatures:
+        - feature: "fake.attribute"
+          matchExpressions:
+            "attr_1": {op: IsTrue}
+            "attr_2": {op: IsTrue}

test/e2e/node_feature_discovery.go

@@ -33,6 +33,7 @@ import (
 	extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
+	taintutils "k8s.io/kubernetes/pkg/util/taints"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2elog "k8s.io/kubernetes/test/e2e/framework"
 	e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
@@ -47,9 +48,31 @@ import (
 )
 
 var (
-	dockerRepo  = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from")
-	dockerTag   = flag.String("nfd.tag", "master", "Docker tag to use")
-	dockerImage = fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)
+	dockerRepo      = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from")
+	dockerTag       = flag.String("nfd.tag", "master", "Docker tag to use")
+	dockerImage     = fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)
+	testTolerations = []corev1.Toleration{
+		{
+			Key:    "nfd.node.kubernetes.io/fake-special-node",
+			Value:  "exists",
+			Effect: "NoExecute",
+		},
+		{
+			Key:    "nfd.node.kubernetes.io/fake-dedicated-node",
+			Value:  "true",
+			Effect: "NoExecute",
+		},
+		{
+			Key:    "nfd.node.kubernetes.io/performance-optimized-node",
+			Value:  "true",
+			Effect: "NoExecute",
+		},
+		{
+			Key:    "nfd.node.kubernetes.io/foo",
+			Value:  "true",
+			Effect: "NoExecute",
+		},
+	}
 )
 
 // cleanupNode deletes all NFD-related metadata from the Node object, i.e.
@@ -82,11 +105,22 @@ func cleanupNode(cs clientset.Interface) {
 				}
 			}
 
+			// Remove taints
+			for _, taint := range node.Spec.Taints {
+				if strings.HasPrefix(taint.Key, nfdv1alpha1.AnnotationNs) {
+					newTaints, removed := taintutils.DeleteTaint(node.Spec.Taints, &taint)
+					if removed {
+						node.Spec.Taints = newTaints
+						update = true
+					}
+				}
+			}
+
 			if !update {
 				break
 			}
 
-			By("Deleting NFD labels and annotations from node " + node.Name)
+			By("Deleting NFD labels, annotations and taints from node " + node.Name)
 			_, err = cs.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{})
 			if err != nil {
 				time.Sleep(100 * time.Millisecond)
@@ -152,8 +186,13 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 
 			// Launch nfd-master
 			By("Creating nfd master pod and nfd-master service")
-			imageOpt := testpod.SpecWithContainerImage(dockerImage)
-			masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt))
+
+			imageOpt := []testpod.SpecOption{
+				testpod.SpecWithContainerImage(dockerImage),
+				testpod.SpecWithTolerations(testTolerations),
+				testpod.SpecWithContainerExtraArgs("-enable-taints"),
+			}
+			masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt...))
 
 			// Create nfd-master service
 			nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name)
@@ -200,6 +239,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 					testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever),
 					testpod.SpecWithContainerImage(dockerImage),
 					testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"),
+					testpod.SpecWithTolerations(testTolerations),
 				}
 				workerPod := testpod.NFDWorker(podSpecOpts...)
 				workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{})
@@ -247,7 +287,10 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 				fConf := cfg.DefaultFeatures
 
 				By("Creating nfd-worker daemonset")
-				podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(dockerImage)}
+				podSpecOpts := []testpod.SpecOption{
+					testpod.SpecWithContainerImage(dockerImage),
+					testpod.SpecWithTolerations(testTolerations),
+				}
 				workerDS := testds.NFDWorker(podSpecOpts...)
 				workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
 				Expect(err).NotTo(HaveOccurred())
@@ -258,6 +301,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 				By("Getting node objects")
 				nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 				Expect(err).NotTo(HaveOccurred())
+				Expect(len(nodeList.Items)).ToNot(BeZero())
 
 				for _, node := range nodeList.Items {
 					nodeConf := testutils.FindNodeConfig(cfg, node.Name)
@@ -381,6 +425,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 					testpod.SpecWithContainerImage(dockerImage),
 					testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")),
 					testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")),
+					testpod.SpecWithTolerations(testTolerations),
 				}
 				workerDS := testds.NFDWorker(podSpecOpts...)
 
@@ -442,6 +487,7 @@ core:
 				podSpecOpts := []testpod.SpecOption{
 					testpod.SpecWithContainerImage(dockerImage),
 					testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"),
+					testpod.SpecWithTolerations(testTolerations),
 				}
 				workerDS := testds.NFDWorker(podSpecOpts...)
 				workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
@@ -471,22 +517,98 @@ core:
 
 				By("Verifying node labels from NodeFeatureRules #1 and #2")
 				Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred())
+
+				// Add features from NodeFeatureRule #3
+				By("Creating NodeFeatureRules #3")
+				Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-3.yaml")).NotTo(HaveOccurred())
+
+				By("Verifying node taints and annotation from NodeFeatureRules #3")
+				expectedTaints := []corev1.Taint{
+					{
+						Key:    "nfd.node.kubernetes.io/fake-special-node",
+						Value:  "exists",
+						Effect: "PreferNoSchedule",
+					},
+					{
+						Key:    "nfd.node.kubernetes.io/fake-dedicated-node",
+						Value:  "true",
+						Effect: "NoExecute",
+					},
+					{
+						Key:    "nfd.node.kubernetes.io/performance-optimized-node",
+						Value:  "true",
+						Effect: "NoExecute",
+					},
+				}
+				expectedAnnotation := map[string]string{
+					"nfd.node.kubernetes.io/taints": "nfd.node.kubernetes.io/fake-special-node=exists:NoExecute,nfd.node.kubernetes.io/fake-dedicated-node=true:NoExecute,nfd.node.kubernetes.io/performance-optimized-node=true:NoExecute"}
+				Expect(waitForNfdNodeTaints(f.ClientSet, expectedTaints)).NotTo(HaveOccurred())
+				Expect(waitForNfdNodeAnnotations(f.ClientSet, expectedAnnotation)).NotTo(HaveOccurred())
+
+				By("Re-applying NodeFeatureRules #3 with updated taints")
+				Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-3-updated.yaml")).NotTo(HaveOccurred())
+				expectedTaintsUpdated := []corev1.Taint{
+					{
+						Key:    "nfd.node.kubernetes.io/fake-special-node",
+						Value:  "exists",
+						Effect: "PreferNoSchedule",
+					},
+					{
+						Key:    "nfd.node.kubernetes.io/foo",
+						Value:  "true",
+						Effect: "NoExecute",
+					},
+				}
+				expectedAnnotationUpdated := map[string]string{
+					"nfd.node.kubernetes.io/taints": "nfd.node.kubernetes.io/fake-special-node=exists:NoExecute,nfd.node.kubernetes.io/foo=true:NoExecute"}
+
+				By("Verifying updated node taints and annotation from NodeFeatureRules #3")
+				Expect(waitForNfdNodeTaints(f.ClientSet, expectedTaintsUpdated)).NotTo(HaveOccurred())
+				Expect(waitForNfdNodeAnnotations(f.ClientSet, expectedAnnotationUpdated)).NotTo(HaveOccurred())
 			})
 		})
 	})
 })
 
+// waitForNfdNodeAnnotations waits for node to be annotated as expected.
+func waitForNfdNodeAnnotations(cli clientset.Interface, expectedAnnotations map[string]string) error {
+	poll := func() error {
+		nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+		if err != nil {
+			return err
+		}
+		for _, node := range nodeList.Items {
+			for k, v := range expectedAnnotations {
+				if diff := cmp.Diff(v, node.Annotations[k]); diff != "" {
+					return fmt.Errorf("node %q annotation does not match expected, diff (expected vs. received): %s", node.Name, diff)
+				}
+			}
+		}
+		return nil
+	}
+
+	// Simple and stupid re-try loop
+	var err error
+	for retry := 0; retry < 3; retry++ {
+		if err = poll(); err == nil {
+			return nil
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return err
+}
+
 // waitForNfdNodeLabels waits for node to be labeled as expected.
-func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) error {
+func waitForNfdNodeLabels(cli clientset.Interface, expectedLabels map[string]string) error {
 	poll := func() error {
 		nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
 		if err != nil {
 			return err
 		}
 		for _, node := range nodeList.Items {
 			labels := nfdLabels(node.Labels)
-			if !cmp.Equal(expected, labels) {
-				return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expected, labels))
+			if !cmp.Equal(expectedLabels, labels) {
+				return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expectedLabels, labels))
 			}
 		}
 		return nil
@@ -503,6 +625,50 @@ func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) e
 	return err
 }
 
+// waitForNfdNodeTaints waits for node to be tainted as expected.
+func waitForNfdNodeTaints(cli clientset.Interface, expectedTaints []corev1.Taint) error {
+	poll := func() error {
+		nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+		if err != nil {
+			return err
+		}
+		for _, node := range nodeList.Items {
+			taints, err := nfdTaints(node.Spec.Taints, node)
+			if err != nil {
+				return fmt.Errorf("failed to fetch nfd owned taints for node: %s", node.Name)
+			}
+			if diff := cmp.Diff(expectedTaints, taints); diff != "" {
+				return fmt.Errorf("node %q taints do not match expected, diff (expected vs. received): %s", node.Name, diff)
+			}
+		}
+		return nil
+	}
+	// Simple and stupid re-try loop
+	var err error
+	for retry := 0; retry < 3; retry++ {
+		if err = poll(); err == nil {
+			return nil
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return err
+}
+
+// nfdTaints returns taints that are owned by the nfd.
+func nfdTaints(taints []corev1.Taint, node corev1.Node) ([]corev1.Taint, error) {
+	// De-serialize the taints annotation into corev1.Taint type for comparision below.
+	var err error
+	nfdTaints := []corev1.Taint{}
+	if val, ok := node.Annotations[nfdv1alpha1.NodeTaintsAnnotation]; ok {
+		sts := strings.Split(val, ",")
+		nfdTaints, _, err = taintutils.ParseTaints(sts)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return nfdTaints, nil
+}
+
 // nfdLabels gets labels that are in the nfd label namespace.
 func nfdLabels(labels map[string]string) map[string]string {
 	ret := map[string]string{}
@@ -513,5 +679,4 @@ func nfdLabels(labels map[string]string) map[string]string {
 		}
 	}
 	return ret
-
 }

test/e2e/utils/pod/pod.go

@@ -219,6 +219,14 @@ func SpecWithMasterNodeSelector(args ...string) SpecOption {
 	}
 }
 
+// SpecWithTolerations returns a SpecOption that modifies the pod to
+// be run on a node with NodeFeatureRule taints.
+func SpecWithTolerations(tolerations []corev1.Toleration) SpecOption {
+	return func(spec *corev1.PodSpec) {
+		spec.Tolerations = append(spec.Tolerations, tolerations...)
+	}
+}
+
 // SpecWithConfigMap returns a SpecOption that mounts a configmap to the first container.
 func SpecWithConfigMap(name, mountPath string) SpecOption {
 	return func(spec *corev1.PodSpec) {