Merge pull request #1016 from marquiz/devel/e2e-nodefeature

test/e2e: more comprehensive test for NodeFeature objects

test/e2e/data/nodefeature-1.yaml

@@ -0,0 +1,31 @@
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeature
+metadata:
+  # This name should ensure that it's processed later than that from nfd-worker
+  name: zzz-e2e-features-1
+spec:
+  # Features for NodeFeatureRule matching
+  features:
+    flags:
+      e2e.flags:
+        elements:
+          flag_1: {}
+          flag_2: {}
+    attributes:
+      # Override features from the fake sources
+      fake.attribute:
+        elements:
+          attr_2: "true"
+    instances:
+      # Append to features from the fake sources
+      fake.instance:
+        elements:
+        - attributes:
+            attr_1: "true"
+            attr_2: "9"
+  # Labels to be created
+  labels:
+    e2e-nodefeature-test-1: "obj-1"
+    e2e-nodefeature-test-2: "obj-1"
+    # Override feature from nfd-worker
+    fake-fakefeature3: "overridden"

test/e2e/data/nodefeature-2.yaml

@@ -0,0 +1,8 @@
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeature
+metadata:
+  name: zzz-e2e-features-2
+spec:
+  labels:
+    e2e-nodefeature-test-1: "overridden-from-obj-2"
+    e2e-nodefeature-test-3: "obj-2"

test/e2e/node_feature_discovery.go

@@ -494,6 +494,94 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
 				})
 			})
 
+			//
+			// Test NodeFeature
+			//
+			Context("and NodeFeature objects deployed", func() {
+				It("labels from the NodeFeature objects should be created", func() {
+					if !useNodeFeatureApi {
+						Skip("NodeFeature API not enabled")
+					}
+
+					// We pick one node targeted for our NodeFeature objects
+					nodes, err := getNonControlPlaneNodes(f.ClientSet)
+					Expect(err).NotTo(HaveOccurred())
+
+					targetNodeName := nodes[0].Name
+					Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
+
+					By("Creating NodeFeature object")
+					nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Verifying node labels from NodeFeature object #1")
+					expectedLabels := map[string]k8sLabels{
+						targetNodeName: {
+							nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-1": "obj-1",
+							nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-2": "obj-1",
+							nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3":      "overridden",
+						},
+					}
+					Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
+
+					By("Deleting NodeFeature object")
+					err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(context.TODO(), nodeFeatures[0], metav1.DeleteOptions{})
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Verifying node labels from NodeFeature object were removed")
+					Expect(waitForNfdNodeLabels(f.ClientSet, nil)).NotTo(HaveOccurred())
+
+					By("Creating nfd-worker daemonset")
+					podSpecOpts := createPodSpecOpts(
+						testpod.SpecWithContainerImage(dockerImage),
+						testpod.SpecWithContainerExtraArgs("-label-sources=fake"),
+					)
+					workerDS := testds.NFDWorker(podSpecOpts...)
+					workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Waiting for worker daemonset pods to be ready")
+					Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred())
+
+					By("Verifying node labels from nfd-worker")
+					expectedLabels = map[string]k8sLabels{
+						"*": {
+							nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1": "true",
+							nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true",
+							nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "true",
+						},
+					}
+					Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
+
+					By("Re-creating NodeFeature object")
+					_, err = testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Verifying node labels from NodeFeature object #1 are created")
+					expectedLabels[targetNodeName] = k8sLabels{
+						nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-1": "obj-1",
+						nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-2": "obj-1",
+						nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1":      "true",
+						nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2":      "true",
+						nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3":      "overridden",
+					}
+					Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
+
+					By("Creating extra namespace")
+					extraNs, err := f.CreateNamespace("node-feature-discvery-extra-ns", nil)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Create NodeFeature object in the extra namespace")
+					_, err = testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-2.yaml", extraNs.Name, targetNodeName)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Verifying node labels from NodeFeature object #2 are created")
+					expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-1"] = "overridden-from-obj-2"
+					expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-3"] = "obj-2"
+					Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
+				})
+			})
+
 			//
 			// Test NodeFeatureRule
 			//
@@ -522,10 +610,13 @@ core:
 					By("Waiting for daemonset pods to be ready")
 					Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred())
 
-					expected := map[string]string{
-						"feature.node.kubernetes.io/e2e-flag-test-1":      "true",
-						"feature.node.kubernetes.io/e2e-attribute-test-1": "true",
-						"feature.node.kubernetes.io/e2e-instance-test-1":  "true"}
+					expected := map[string]k8sLabels{
+						"*": {
+							nfdv1alpha1.FeatureLabelNs + "/e2e-flag-test-1":      "true",
+							nfdv1alpha1.FeatureLabelNs + "/e2e-attribute-test-1": "true",
+							nfdv1alpha1.FeatureLabelNs + "/e2e-instance-test-1":  "true",
+						},
+					}
 
 					By("Creating NodeFeatureRules #1")
 					Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred())
@@ -537,9 +628,9 @@ core:
 					Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred())
 
 					// Add features from NodeFeatureRule #2
-					expected["feature.node.kubernetes.io/e2e-matchany-test-1"] = "true"
-					expected["feature.node.kubernetes.io/e2e-template-test-1-instance_1"] = "found"
-					expected["feature.node.kubernetes.io/e2e-template-test-1-instance_2"] = "found"
+					expected["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-matchany-test-1"] = "true"
+					expected["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_1"] = "found"
+					expected["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_2"] = "found"
 
 					By("Verifying node labels from NodeFeatureRules #1 and #2")
 					Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred())
@@ -636,17 +727,26 @@ func waitForNfdNodeAnnotations(cli clientset.Interface, expected map[string]stri
 	return err
 }
 
+type k8sLabels map[string]string
+
 // waitForNfdNodeLabels waits for node to be labeled as expected.
-func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) error {
+func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]k8sLabels) error {
 	poll := func() error {
 		nodes, err := getNonControlPlaneNodes(cli)
 		if err != nil {
 			return err
 		}
 		for _, node := range nodes {
 			labels := nfdLabels(node.Labels)
-			if !cmp.Equal(expected, labels) {
-				return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expected, labels))
+			nodeExpected, ok := expected[node.Name]
+			if !ok {
+				nodeExpected = k8sLabels{}
+				if defaultExpected, ok := expected["*"]; ok {
+					nodeExpected = defaultExpected
+				}
+			}
+			if !cmp.Equal(nodeExpected, labels) {
+				return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(nodeExpected, labels))
 			}
 		}
 		return nil
@@ -733,8 +833,8 @@ func getNonControlPlaneNodes(cli clientset.Interface) ([]corev1.Node, error) {
 }
 
 // nfdLabels gets labels that are in the nfd label namespace.
-func nfdLabels(labels map[string]string) map[string]string {
-	ret := map[string]string{}
+func nfdLabels(labels map[string]string) k8sLabels {
+	ret := k8sLabels{}
 
 	for key, val := range labels {
 		if strings.HasPrefix(key, nfdv1alpha1.FeatureLabelNs) {

test/e2e/utils/crd.go

@@ -59,6 +59,38 @@ func CreateNfdCRDs(cli extclient.Interface) ([]*apiextensionsv1.CustomResourceDe
 	return newCRDs, nil
 }
 
+// CreateOrUpdateNodeFeaturesFromFile creates/updates a NodeFeature object from a given file located under test data directory.
+func CreateOrUpdateNodeFeaturesFromFile(cli nfdclientset.Interface, filename, namespace, nodename string) ([]string, error) {
+	objs, err := nodeFeaturesFromFile(filepath.Join(packagePath, "..", "data", filename))
+	if err != nil {
+		return nil, err
+	}
+
+	names := make([]string, len(objs))
+	for i, obj := range objs {
+		obj.Namespace = namespace
+		if obj.Labels == nil {
+			obj.Labels = map[string]string{}
+		}
+		obj.Labels[nfdv1alpha1.NodeFeatureObjNodeNameLabel] = nodename
+
+		if oldObj, err := cli.NfdV1alpha1().NodeFeatures(namespace).Get(context.TODO(), obj.Name, metav1.GetOptions{}); errors.IsNotFound(err) {
+			if _, err := cli.NfdV1alpha1().NodeFeatures(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}); err != nil {
+				return names, fmt.Errorf("failed to create NodeFeature %w", err)
+			}
+		} else if err == nil {
+			obj.SetResourceVersion(oldObj.GetResourceVersion())
+			if _, err = cli.NfdV1alpha1().NodeFeatures(namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}); err != nil {
+				return names, fmt.Errorf("failed to update NodeFeature object: %w", err)
+			}
+		} else {
+			return names, fmt.Errorf("failed to get NodeFeature %w", err)
+		}
+		names[i] = obj.Name
+	}
+	return names, nil
+}
+
 // CreateNodeFeatureRuleFromFile creates a NodeFeatureRule object from a given file located under test data directory.
 func CreateNodeFeatureRulesFromFile(cli nfdclientset.Interface, filename string) error {
 	objs, err := nodeFeatureRulesFromFile(filepath.Join(packagePath, "..", "data", filename))
@@ -139,6 +171,25 @@ func crdsFromFile(path string) ([]*apiextensionsv1.CustomResourceDefinition, err
 	return crds, nil
 }
 
+func nodeFeaturesFromFile(path string) ([]*nfdv1alpha1.NodeFeature, error) {
+	objs, err := apiObjsFromFile(path, nfdscheme.Codecs.UniversalDeserializer())
+	if err != nil {
+		return nil, err
+	}
+
+	crs := make([]*nfdv1alpha1.NodeFeature, len(objs))
+
+	for i, obj := range objs {
+		var ok bool
+		crs[i], ok = obj.(*nfdv1alpha1.NodeFeature)
+		if !ok {
+			return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path)
+		}
+	}
+
+	return crs, nil
+}
+
 func nodeFeatureRulesFromFile(path string) ([]*nfdv1alpha1.NodeFeatureRule, error) {
 	objs, err := apiObjsFromFile(path, nfdscheme.Codecs.UniversalDeserializer())
 	if err != nil {