diff --git a/test/e2e/data/nodefeature-1.yaml b/test/e2e/data/nodefeature-1.yaml new file mode 100644 index 0000000000..4b3b965235 --- /dev/null +++ b/test/e2e/data/nodefeature-1.yaml @@ -0,0 +1,31 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeature +metadata: + # This name should ensure that it's processed later than that from nfd-worker + name: zzz-e2e-features-1 +spec: + # Features for NodeFeatureRule matching + features: + flags: + e2e.flags: + elements: + flag_1: {} + flag_2: {} + attributes: + # Override features from the fake sources + fake.attribute: + elements: + attr_2: "true" + instances: + # Append to features from the fake sources + fake.instance: + elements: + - attributes: + attr_1: "true" + attr_2: "9" + # Labels to be created + labels: + e2e-nodefeature-test-1: "obj-1" + e2e-nodefeature-test-2: "obj-1" + # Override feature from nfd-worker + fake-fakefeature3: "overridden" diff --git a/test/e2e/data/nodefeature-2.yaml b/test/e2e/data/nodefeature-2.yaml new file mode 100644 index 0000000000..0696902d3f --- /dev/null +++ b/test/e2e/data/nodefeature-2.yaml @@ -0,0 +1,8 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeature +metadata: + name: zzz-e2e-features-2 +spec: + labels: + e2e-nodefeature-test-1: "overridden-from-obj-2" + e2e-nodefeature-test-3: "obj-2" diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 5ba26404a8..2553e30eca 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -494,6 +494,94 @@ var _ = SIGDescribe("Node Feature Discovery", func() { }) }) + // + // Test NodeFeature + // + Context("and NodeFeature objects deployed", func() { + It("labels from the NodeFeature objects should be created", func() { + if !useNodeFeatureApi { + Skip("NodeFeature API not enabled") + } + + // We pick one node targeted for our NodeFeature objects + nodes, err := getNonControlPlaneNodes(f.ClientSet) + Expect(err).NotTo(HaveOccurred()) + + targetNodeName := nodes[0].Name + Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found") + + By("Creating NodeFeature object") + nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeature object #1") + expectedLabels := map[string]k8sLabels{ + targetNodeName: { + nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-1": "obj-1", + nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-2": "obj-1", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "overridden", + }, + } + Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred()) + + By("Deleting NodeFeature object") + err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(context.TODO(), nodeFeatures[0], metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeature object were removed") + Expect(waitForNfdNodeLabels(f.ClientSet, nil)).NotTo(HaveOccurred()) + + By("Creating nfd-worker daemonset") + podSpecOpts := createPodSpecOpts( + testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithContainerExtraArgs("-label-sources=fake"), + ) + workerDS := testds.NFDWorker(podSpecOpts...) + workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for worker daemonset pods to be ready") + Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + + By("Verifying node labels from nfd-worker") + expectedLabels = map[string]k8sLabels{ + "*": { + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1": "true", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "true", + }, + } + Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred()) + + By("Re-creating NodeFeature object") + _, err = testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeature object #1 are created") + expectedLabels[targetNodeName] = k8sLabels{ + nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-1": "obj-1", + nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-2": "obj-1", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1": "true", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "overridden", + } + Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred()) + + By("Creating extra namespace") + extraNs, err := f.CreateNamespace("node-feature-discvery-extra-ns", nil) + Expect(err).NotTo(HaveOccurred()) + + By("Create NodeFeature object in the extra namespace") + _, err = testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-2.yaml", extraNs.Name, targetNodeName) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeature object #2 are created") + expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-1"] = "overridden-from-obj-2" + expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-3"] = "obj-2" + Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred()) + }) + }) + // // Test NodeFeatureRule // @@ -522,10 +610,13 @@ core: By("Waiting for daemonset pods to be ready") Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) - expected := map[string]string{ - "feature.node.kubernetes.io/e2e-flag-test-1": "true", - "feature.node.kubernetes.io/e2e-attribute-test-1": "true", - "feature.node.kubernetes.io/e2e-instance-test-1": "true"} + expected := map[string]k8sLabels{ + "*": { + nfdv1alpha1.FeatureLabelNs + "/e2e-flag-test-1": "true", + nfdv1alpha1.FeatureLabelNs + "/e2e-attribute-test-1": "true", + nfdv1alpha1.FeatureLabelNs + "/e2e-instance-test-1": "true", + }, + } By("Creating NodeFeatureRules #1") Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred()) @@ -537,9 +628,9 @@ core: Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred()) // Add features from NodeFeatureRule #2 - expected["feature.node.kubernetes.io/e2e-matchany-test-1"] = "true" - expected["feature.node.kubernetes.io/e2e-template-test-1-instance_1"] = "found" - expected["feature.node.kubernetes.io/e2e-template-test-1-instance_2"] = "found" + expected["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-matchany-test-1"] = "true" + expected["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_1"] = "found" + expected["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_2"] = "found" By("Verifying node labels from NodeFeatureRules #1 and #2") Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) @@ -636,8 +727,10 @@ func waitForNfdNodeAnnotations(cli clientset.Interface, expected map[string]stri return err } +type k8sLabels map[string]string + // waitForNfdNodeLabels waits for node to be labeled as expected. -func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) error { +func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]k8sLabels) error { poll := func() error { nodes, err := getNonControlPlaneNodes(cli) if err != nil { @@ -645,8 +738,15 @@ func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) e } for _, node := range nodes { labels := nfdLabels(node.Labels) - if !cmp.Equal(expected, labels) { - return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expected, labels)) + nodeExpected, ok := expected[node.Name] + if !ok { + nodeExpected = k8sLabels{} + if defaultExpected, ok := expected["*"]; ok { + nodeExpected = defaultExpected + } + } + if !cmp.Equal(nodeExpected, labels) { + return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(nodeExpected, labels)) } } return nil @@ -733,8 +833,8 @@ func getNonControlPlaneNodes(cli clientset.Interface) ([]corev1.Node, error) { } // nfdLabels gets labels that are in the nfd label namespace. -func nfdLabels(labels map[string]string) map[string]string { - ret := map[string]string{} +func nfdLabels(labels map[string]string) k8sLabels { + ret := k8sLabels{} for key, val := range labels { if strings.HasPrefix(key, nfdv1alpha1.FeatureLabelNs) { diff --git a/test/e2e/utils/crd.go b/test/e2e/utils/crd.go index 6d95d90668..6546f312af 100644 --- a/test/e2e/utils/crd.go +++ b/test/e2e/utils/crd.go @@ -59,6 +59,38 @@ func CreateNfdCRDs(cli extclient.Interface) ([]*apiextensionsv1.CustomResourceDe return newCRDs, nil } +// CreateOrUpdateNodeFeaturesFromFile creates/updates a NodeFeature object from a given file located under test data directory. +func CreateOrUpdateNodeFeaturesFromFile(cli nfdclientset.Interface, filename, namespace, nodename string) ([]string, error) { + objs, err := nodeFeaturesFromFile(filepath.Join(packagePath, "..", "data", filename)) + if err != nil { + return nil, err + } + + names := make([]string, len(objs)) + for i, obj := range objs { + obj.Namespace = namespace + if obj.Labels == nil { + obj.Labels = map[string]string{} + } + obj.Labels[nfdv1alpha1.NodeFeatureObjNodeNameLabel] = nodename + + if oldObj, err := cli.NfdV1alpha1().NodeFeatures(namespace).Get(context.TODO(), obj.Name, metav1.GetOptions{}); errors.IsNotFound(err) { + if _, err := cli.NfdV1alpha1().NodeFeatures(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}); err != nil { + return names, fmt.Errorf("failed to create NodeFeature %w", err) + } + } else if err == nil { + obj.SetResourceVersion(oldObj.GetResourceVersion()) + if _, err = cli.NfdV1alpha1().NodeFeatures(namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}); err != nil { + return names, fmt.Errorf("failed to update NodeFeature object: %w", err) + } + } else { + return names, fmt.Errorf("failed to get NodeFeature %w", err) + } + names[i] = obj.Name + } + return names, nil +} + // CreateNodeFeatureRuleFromFile creates a NodeFeatureRule object from a given file located under test data directory. func CreateNodeFeatureRulesFromFile(cli nfdclientset.Interface, filename string) error { objs, err := nodeFeatureRulesFromFile(filepath.Join(packagePath, "..", "data", filename)) @@ -139,6 +171,25 @@ func crdsFromFile(path string) ([]*apiextensionsv1.CustomResourceDefinition, err return crds, nil } +func nodeFeaturesFromFile(path string) ([]*nfdv1alpha1.NodeFeature, error) { + objs, err := apiObjsFromFile(path, nfdscheme.Codecs.UniversalDeserializer()) + if err != nil { + return nil, err + } + + crs := make([]*nfdv1alpha1.NodeFeature, len(objs)) + + for i, obj := range objs { + var ok bool + crs[i], ok = obj.(*nfdv1alpha1.NodeFeature) + if !ok { + return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path) + } + } + + return crs, nil +} + func nodeFeatureRulesFromFile(path string) ([]*nfdv1alpha1.NodeFeatureRule, error) { objs, err := apiObjsFromFile(path, nfdscheme.Codecs.UniversalDeserializer()) if err != nil {