From fb5039a2e7e723b4c7ca54452115ce4e37616284 Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Thu, 1 Dec 2022 01:21:21 +0200 Subject: [PATCH] Add nfd E2E tests for tainting feature Extend current E2E tests to check tainting feature of nfd implemented in https://github.com/kubernetes-sigs/node-feature-discovery/pull/910 Signed-off-by: Feruzjon Muyassarov --- test/e2e/data/nodefeaturerule-3-updated.yaml | 32 ++++ test/e2e/data/nodefeaturerule-3.yaml | 35 ++++ test/e2e/node_feature_discovery.go | 180 ++++++++++++++++++- test/e2e/utils/crd.go | 21 +++ test/e2e/utils/pod/pod.go | 8 + 5 files changed, 269 insertions(+), 7 deletions(-) create mode 100644 test/e2e/data/nodefeaturerule-3-updated.yaml create mode 100644 test/e2e/data/nodefeaturerule-3.yaml diff --git a/test/e2e/data/nodefeaturerule-3-updated.yaml b/test/e2e/data/nodefeaturerule-3-updated.yaml new file mode 100644 index 0000000000..8b17789788 --- /dev/null +++ b/test/e2e/data/nodefeaturerule-3-updated.yaml @@ -0,0 +1,32 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: e2e-test-3 +spec: + rules: + # Positive test expected to set the taints + - name: "e2e-taint-test-1" + taints: + - effect: PreferNoSchedule + key: "nfd.node.kubernetes.io/fake-special-node" + value: "exists" + - effect: NoExecute + key: "nfd.node.kubernetes.io/foo" + value: "true" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsFalse} + + # Negative test not supposed to set the taints + - name: "e2e-taint-test-2" + taints: + - effect: PreferNoSchedule + key: "nfd.node.kubernetes.io/fake-cpu" + value: "true" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsTrue} diff --git a/test/e2e/data/nodefeaturerule-3.yaml b/test/e2e/data/nodefeaturerule-3.yaml new file mode 100644 index 0000000000..b9701cb196 --- /dev/null +++ b/test/e2e/data/nodefeaturerule-3.yaml @@ -0,0 +1,35 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: e2e-test-3 +spec: + rules: + # Positive test expected to set the taints + - name: "e2e-taint-test-1" + taints: + - effect: PreferNoSchedule + key: "nfd.node.kubernetes.io/fake-special-node" + value: "exists" + - effect: NoExecute + key: "nfd.node.kubernetes.io/fake-dedicated-node" + value: "true" + - effect: "NoExecute" + key: "nfd.node.kubernetes.io/performance-optimized-node" + value: "true" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsFalse} + + # Negative test not supposed to set the taints + - name: "e2e-taint-test-2" + taints: + - effect: PreferNoSchedule + key: "nfd.node.kubernetes.io/fake-special-cpu" + value: "true" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsTrue} diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index f2572313ae..be206d7747 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -49,9 +49,31 @@ import ( ) var ( - dockerRepo = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from") - dockerTag = flag.String("nfd.tag", "master", "Docker tag to use") - dockerImage = fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) + dockerRepo = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from") + dockerTag = flag.String("nfd.tag", "master", "Docker tag to use") + dockerImage = fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) + testTolerations = []corev1.Toleration{ + { + Key: "nfd.node.kubernetes.io/fake-special-node", + Value: "exists", + Effect: "NoExecute", + }, + { + Key: "nfd.node.kubernetes.io/fake-dedicated-node", + Value: "true", + Effect: "NoExecute", + }, + { + Key: "nfd.node.kubernetes.io/performance-optimized-node", + Value: "true", + Effect: "NoExecute", + }, + { + Key: "nfd.node.kubernetes.io/foo", + Value: "true", + Effect: "NoExecute", + }, + } ) // cleanupNode deletes all NFD-related metadata from the Node object, i.e. @@ -84,11 +106,22 @@ func cleanupNode(cs clientset.Interface) { } } + // Remove taints + for _, taint := range node.Spec.Taints { + if strings.HasPrefix(taint.Key, nfdv1alpha1.AnnotationNs) { + newTaints, removed := taintutils.DeleteTaint(node.Spec.Taints, &taint) + if removed { + node.Spec.Taints = newTaints + update = true + } + } + } + if !update { break } - By("Deleting NFD labels and annotations from node " + node.Name) + By("Deleting NFD labels, annotations and taints from node " + node.Name) _, err = cs.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) if err != nil { time.Sleep(100 * time.Millisecond) @@ -162,8 +195,13 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-master By("Creating nfd master pod and nfd-master service") - imageOpt := testpod.SpecWithContainerImage(dockerImage) - masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt)) + + imageOpt := []testpod.SpecOption{ + testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithTolerations(testTolerations), + testpod.SpecWithContainerExtraArgs("-enable-taints"), + } + masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt...)) // Create nfd-master service nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) @@ -210,6 +248,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever), testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"), + testpod.SpecWithTolerations(testTolerations), } workerPod := testpod.NFDWorker(podSpecOpts...) workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{}) @@ -257,7 +296,10 @@ var _ = SIGDescribe("Node Feature Discovery", func() { fConf := cfg.DefaultFeatures By("Creating nfd-worker daemonset") - podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(dockerImage)} + podSpecOpts := []testpod.SpecOption{ + testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithTolerations(testTolerations), + } workerDS := testds.NFDWorker(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -268,6 +310,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { By("Getting node objects") nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeList.Items)).ToNot(BeZero()) for _, node := range nodeList.Items { nodeConf := testutils.FindNodeConfig(cfg, node.Name) @@ -384,6 +427,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")), testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")), + testpod.SpecWithTolerations(testTolerations), } workerDS := testds.NFDWorker(podSpecOpts...) @@ -445,6 +489,7 @@ core: podSpecOpts := []testpod.SpecOption{ testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), + testpod.SpecWithTolerations(testTolerations), } workerDS := testds.NFDWorker(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) @@ -474,11 +519,87 @@ core: By("Verifying node labels from NodeFeatureRules #1 and #2") Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) + + // Add features from NodeFeatureRule #3 + By("Creating NodeFeatureRules #3") + Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-3.yaml")).NotTo(HaveOccurred()) + + By("Verifying node taints and annotation from NodeFeatureRules #3") + expectedTaints := []corev1.Taint{ + { + Key: "nfd.node.kubernetes.io/fake-special-node", + Value: "exists", + Effect: "PreferNoSchedule", + }, + { + Key: "nfd.node.kubernetes.io/fake-dedicated-node", + Value: "true", + Effect: "NoExecute", + }, + { + Key: "nfd.node.kubernetes.io/performance-optimized-node", + Value: "true", + Effect: "NoExecute", + }, + } + expectedAnnotation := map[string]string{ + "nfd.node.kubernetes.io/taints": "nfd.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,nfd.node.kubernetes.io/fake-dedicated-node=true:NoExecute,nfd.node.kubernetes.io/performance-optimized-node=true:NoExecute"} + Expect(waitForNfdNodeTaints(f.ClientSet, expectedTaints)).NotTo(HaveOccurred()) + Expect(waitForNfdNodeAnnotations(f.ClientSet, expectedAnnotation)).NotTo(HaveOccurred()) + + By("Re-applying NodeFeatureRules #3 with updated taints") + Expect(testutils.UpdateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-3-updated.yaml")).NotTo(HaveOccurred()) + expectedTaintsUpdated := []corev1.Taint{ + { + Key: "nfd.node.kubernetes.io/fake-special-node", + Value: "exists", + Effect: "PreferNoSchedule", + }, + { + Key: "nfd.node.kubernetes.io/foo", + Value: "true", + Effect: "NoExecute", + }, + } + expectedAnnotationUpdated := map[string]string{ + "nfd.node.kubernetes.io/taints": "nfd.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,nfd.node.kubernetes.io/foo=true:NoExecute"} + + By("Verifying updated node taints and annotation from NodeFeatureRules #3") + Expect(waitForNfdNodeTaints(f.ClientSet, expectedTaintsUpdated)).NotTo(HaveOccurred()) + Expect(waitForNfdNodeAnnotations(f.ClientSet, expectedAnnotationUpdated)).NotTo(HaveOccurred()) }) }) }) }) +// waitForNfdNodeAnnotations waits for node to be annotated as expected. +func waitForNfdNodeAnnotations(cli clientset.Interface, expected map[string]string) error { + poll := func() error { + nodes, err := getNonControlPlaneNodes(cli) + if err != nil { + return err + } + for _, node := range nodes { + for k, v := range expected { + if diff := cmp.Diff(v, node.Annotations[k]); diff != "" { + return fmt.Errorf("node %q annotation does not match expected, diff (expected vs. received): %s", node.Name, diff) + } + } + } + return nil + } + + // Simple and stupid re-try loop + var err error + for retry := 0; retry < 3; retry++ { + if err = poll(); err == nil { + return nil + } + time.Sleep(2 * time.Second) + } + return err +} + // waitForNfdNodeLabels waits for node to be labeled as expected. func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) error { poll := func() error { @@ -506,6 +627,51 @@ func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) e return err } +// waitForNfdNodeTaints waits for node to be tainted as expected. +func waitForNfdNodeTaints(cli clientset.Interface, expected []corev1.Taint) error { + poll := func() error { + nodes, err := getNonControlPlaneNodes(cli) + if err != nil { + return err + } + for _, node := range nodes { + taints, err := nfdTaints(node.Spec.Taints, node) + if err != nil { + return fmt.Errorf("failed to fetch nfd owned taints for node: %s", node.Name) + } + if !cmp.Equal(expected, taints) { + return fmt.Errorf("node %q taints do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expected, taints)) + } + } + return nil + } + + // Simple and stupid re-try loop + var err error + for retry := 0; retry < 3; retry++ { + if err = poll(); err == nil { + return nil + } + time.Sleep(10 * time.Second) + } + return err +} + +// nfdTaints returns taints that are owned by the nfd. +func nfdTaints(taints []corev1.Taint, node corev1.Node) ([]corev1.Taint, error) { + // De-serialize the taints annotation into corev1.Taint type for comparision below. + var err error + nfdTaints := []corev1.Taint{} + if val, ok := node.Annotations[nfdv1alpha1.NodeTaintsAnnotation]; ok { + sts := strings.Split(val, ",") + nfdTaints, _, err = taintutils.ParseTaints(sts) + if err != nil { + return nil, err + } + } + return nfdTaints, nil +} + // getNonControlPlaneNodes gets the nodes that are not tainted for exclusive control-plane usage func getNonControlPlaneNodes(cli clientset.Interface) ([]corev1.Node, error) { nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) diff --git a/test/e2e/utils/crd.go b/test/e2e/utils/crd.go index 76ea53c00d..6d95d90668 100644 --- a/test/e2e/utils/crd.go +++ b/test/e2e/utils/crd.go @@ -74,6 +74,27 @@ func CreateNodeFeatureRulesFromFile(cli nfdclientset.Interface, filename string) return nil } +// UpdateNodeFeatureRulesFromFile updates existing NodeFeatureRule object from a given file located under test data directory. +func UpdateNodeFeatureRulesFromFile(cli nfdclientset.Interface, filename string) error { + objs, err := nodeFeatureRulesFromFile(filepath.Join(packagePath, "..", "data", filename)) + if err != nil { + return err + } + + for _, obj := range objs { + var nfr *nfdv1alpha1.NodeFeatureRule + if nfr, err = cli.NfdV1alpha1().NodeFeatureRules().Get(context.TODO(), obj.Name, metav1.GetOptions{}); err != nil { + return fmt.Errorf("failed to get NodeFeatureRule %w", err) + } + + obj.SetResourceVersion(nfr.GetResourceVersion()) + if _, err = cli.NfdV1alpha1().NodeFeatureRules().Update(context.TODO(), obj, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update NodeFeatureRule %w", err) + } + } + return nil +} + func apiObjsFromFile(path string, decoder apiruntime.Decoder) ([]apiruntime.Object, error) { data, err := os.ReadFile(path) if err != nil { diff --git a/test/e2e/utils/pod/pod.go b/test/e2e/utils/pod/pod.go index b6396279bd..9cea5630fb 100644 --- a/test/e2e/utils/pod/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -219,6 +219,14 @@ func SpecWithMasterNodeSelector(args ...string) SpecOption { } } +// SpecWithTolerations returns a SpecOption that modifies the pod to +// be run on a node with NodeFeatureRule taints. +func SpecWithTolerations(tolerations []corev1.Toleration) SpecOption { + return func(spec *corev1.PodSpec) { + spec.Tolerations = append(spec.Tolerations, tolerations...) + } +} + // SpecWithConfigMap returns a SpecOption that mounts a configmap to the first container. func SpecWithConfigMap(name, mountPath string) SpecOption { return func(spec *corev1.PodSpec) {