test: implement e2e test of the deny-label-ns flag

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/stretchr/testify v1.8.0
 	github.com/vektra/errors v0.0.0-20140903201135-c64d83aba85a
+	golang.org/x/exp v0.0.0-20230307190834-24139beb5833
 	golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10
 	google.golang.org/grpc v1.49.0
 	google.golang.org/protobuf v1.28.1

go.sum

@@ -732,6 +732,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20230307190834-24139beb5833 h1:SChBja7BCQewoTAU7IgvucQKMIXrEpFxNMs0spT3/5s=
+golang.org/x/exp v0.0.0-20230307190834-24139beb5833/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

test/e2e/data/nodefeature-3.yaml

@@ -0,0 +1,12 @@
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeature
+metadata:
+  name: zzz-e2e-features-3
+spec:
+  labels:
+    random.denied.ns/e2e-nodefeature-test-1: "denied-ns"
+    random.unwanted.ns/e2e-nodefeature-test-2: "unwanted-ns"
+    custom.vendor.io/e2e-nodefeature-test-3: "vendor-ns"
+    kubernetes.io/denied-label: "kubernetes-ns-1"
+    subns.kubernetes.io/denied-label: "kubernetes-ns-2"
+    e2e-nodefeature-test-4: "obj-4"

test/e2e/node_feature_discovery_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"golang.org/x/exp/maps"
 
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -224,6 +225,10 @@ var _ = SIGDescribe("NFD master and worker", func() {
 					testpod.SpecWithContainerImage(dockerImage()),
 					testpod.SpecWithTolerations(testTolerations),
 					testpod.SpecWithContainerExtraArgs("-enable-taints"),
+					testpod.SpecWithContainerExtraArgs(
+						"-deny-label-ns=*.denied.ns,random.unwanted.ns,*.vendor.io",
+						"-extra-label-ns=custom.vendor.io",
+					),
 				)
 				masterPod := e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(podSpecOpts...))
 
@@ -527,14 +532,17 @@ var _ = SIGDescribe("NFD master and worker", func() {
 							nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3":      "overridden",
 						},
 					}
-					Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
-
+					Expect(checkForNodeLabels(f.ClientSet,
+						expectedLabels, nodes,
+					)).NotTo(HaveOccurred())
 					By("Deleting NodeFeature object")
 					err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(context.TODO(), nodeFeatures[0], metav1.DeleteOptions{})
 					Expect(err).NotTo(HaveOccurred())
 
 					By("Verifying node labels from NodeFeature object were removed")
-					Expect(waitForNfdNodeLabels(f.ClientSet, nil)).NotTo(HaveOccurred())
+					Expect(checkForNodeLabels(f.ClientSet,
+						nil, nodes,
+					)).NotTo(HaveOccurred())
 
 					By("Creating nfd-worker daemonset")
 					podSpecOpts := createPodSpecOpts(
@@ -556,7 +564,9 @@ var _ = SIGDescribe("NFD master and worker", func() {
 							nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "true",
 						},
 					}
-					Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
+					Expect(checkForNodeLabels(f.ClientSet,
+						expectedLabels, nodes,
+					)).NotTo(HaveOccurred())
 
 					By("Re-creating NodeFeature object")
 					_, err = testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
@@ -570,7 +580,9 @@ var _ = SIGDescribe("NFD master and worker", func() {
 						nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2":      "true",
 						nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3":      "overridden",
 					}
-					Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
+					Expect(checkForNodeLabels(f.ClientSet,
+						expectedLabels, nodes,
+					)).NotTo(HaveOccurred())
 
 					By("Creating extra namespace")
 					extraNs, err := f.CreateNamespace("node-feature-discvery-extra-ns", nil)
@@ -583,7 +595,7 @@ var _ = SIGDescribe("NFD master and worker", func() {
 					By("Verifying node labels from NodeFeature object #2 are created")
 					expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-1"] = "overridden-from-obj-2"
 					expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-3"] = "obj-2"
-					Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
+					Expect(checkForNodeLabels(f.ClientSet, expectedLabels, nodes)).NotTo(HaveOccurred())
 
 					By("Deleting NodeFeature object from the extra namespace")
 					err = nfdClient.NfdV1alpha1().NodeFeatures(extraNs.Name).Delete(context.TODO(), nodeFeatures[0], metav1.DeleteOptions{})
@@ -592,25 +604,74 @@ var _ = SIGDescribe("NFD master and worker", func() {
 					By("Verifying node labels from NodeFeature object were removed")
 					expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-1"] = "obj-1"
 					delete(expectedLabels[targetNodeName], nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-3")
-					Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
+					Expect(checkForNodeLabels(f.ClientSet, expectedLabels, nodes)).NotTo(HaveOccurred())
+					Expect(checkForNodeLabels(f.ClientSet,
+						expectedLabels,
+						nodes,
+					)).NotTo(HaveOccurred())
+				})
+
+				It("denied labels should not be created by the NodeFeature object", func() {
+					if !useNodeFeatureApi {
+						Skip("NodeFeature API not enabled")
+					}
+
+					nodes, err := getNonControlPlaneNodes(f.ClientSet)
+					Expect(err).NotTo(HaveOccurred())
+
+					targetNodeName := nodes[0].Name
+					Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
+
+					// Apply Node Feature object
+					By("Create NodeFeature object")
+					nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-3.yaml", f.Namespace.Name, targetNodeName)
+					Expect(err).NotTo(HaveOccurred())
+
+					// Verify that denied label was not added
+					By("Verifying that denied labels were not added")
+					expectedLabels := map[string]k8sLabels{
+						targetNodeName: {
+							nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-4": "obj-4",
+							"custom.vendor.io/e2e-nodefeature-test-3":              "vendor-ns",
+						},
+					}
+					Expect(checkForNodeLabels(
+						f.ClientSet,
+						expectedLabels,
+						nodes,
+					)).NotTo(HaveOccurred())
+
+					By("Deleting NodeFeature object")
+					err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(context.TODO(), nodeFeatures[0], metav1.DeleteOptions{})
+					Expect(err).NotTo(HaveOccurred())
+
+					Expect(checkForNodeLabels(
+						f.ClientSet,
+						nil,
+						nodes,
+					)).NotTo(HaveOccurred())
 				})
 			})
 
-			//
 			// Test NodeFeatureRule
 			//
 			Context("and nfd-worker and NodeFeatureRules objects deployed", func() {
 				It("custom labels from the NodeFeatureRule rules should be created", func() {
+					nodes, err := getNonControlPlaneNodes(f.ClientSet)
+					Expect(err).NotTo(HaveOccurred())
+
+					targetNodeName := nodes[0].Name
+					Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")
+
 					By("Creating nfd-worker config")
 					cm := testutils.NewConfigMap("nfd-worker-conf", "nfd-worker.conf", `
 core:
   sleepInterval: "1s"
   featureSources: ["fake"]
   labelSources: []
 `)
-					cm, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm, metav1.CreateOptions{})
+					cm, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm, metav1.CreateOptions{})
 					Expect(err).NotTo(HaveOccurred())
-
 					By("Creating nfd-worker daemonset")
 					podSpecOpts := createPodSpecOpts(
 						testpod.SpecWithContainerImage(dockerImage()),
@@ -636,7 +697,11 @@ core:
 					Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred())
 
 					By("Verifying node labels from NodeFeatureRules #1")
-					Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred())
+					Expect(checkForNodeLabels(
+						f.ClientSet,
+						expected,
+						nodes,
+					)).NotTo(HaveOccurred())
 
 					By("Creating NodeFeatureRules #2")
 					Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred())
@@ -647,7 +712,11 @@ core:
 					expected["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_2"] = "found"
 
 					By("Verifying node labels from NodeFeatureRules #1 and #2")
-					Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred())
+					Expect(checkForNodeLabels(
+						f.ClientSet,
+						expected,
+						nodes,
+					)).NotTo(HaveOccurred())
 
 					// Add features from NodeFeatureRule #3
 					By("Creating NodeFeatureRules #3")
@@ -708,7 +777,6 @@ core:
 	// Run the actual tests
 	Context("when running NFD with gRPC API enabled", func() {
 		nfdTestSuite(false)
-
 	})
 
 	Context("when running NFD with NodeFeature CRD API enabled", func() {
@@ -747,24 +815,29 @@ func waitForNfdNodeAnnotations(cli clientset.Interface, expected map[string]stri
 
 type k8sLabels map[string]string
 
-// waitForNfdNodeLabels waits for node to be labeled as expected.
-func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]k8sLabels) error {
+// checkForNfdNodeLabels waits and checks that node is labeled as expected.
+func checkForNodeLabels(cli clientset.Interface, expectedNewLabels map[string]k8sLabels, oldNodes []corev1.Node) error {
+
 	poll := func() error {
 		nodes, err := getNonControlPlaneNodes(cli)
 		if err != nil {
 			return err
 		}
 		for _, node := range nodes {
-			labels := nfdLabels(node.Labels)
-			nodeExpected, ok := expected[node.Name]
+			nodeExpected, ok := expectedNewLabels[node.Name]
 			if !ok {
 				nodeExpected = k8sLabels{}
-				if defaultExpected, ok := expected["*"]; ok {
+				if defaultExpected, ok := expectedNewLabels["*"]; ok {
 					nodeExpected = defaultExpected
 				}
 			}
-			if !cmp.Equal(nodeExpected, labels) {
-				return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(nodeExpected, labels))
+
+			oldLabels := getNodeLabels(oldNodes, node.Name)
+			expectedNewLabels := maps.Clone(oldLabels)
+			maps.Copy(expectedNewLabels, nodeExpected)
+
+			if !cmp.Equal(node.Labels, expectedNewLabels) {
+				return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expectedNewLabels, node.Labels))
 			}
 		}
 		return nil
@@ -850,15 +923,11 @@ func getNonControlPlaneNodes(cli clientset.Interface) ([]corev1.Node, error) {
 	return out, nil
 }
 
-// nfdLabels gets labels that are in the nfd label namespace.
-func nfdLabels(labels map[string]string) k8sLabels {
-	ret := k8sLabels{}
-
-	for key, val := range labels {
-		if strings.HasPrefix(key, nfdv1alpha1.FeatureLabelNs) {
-			ret[key] = val
+func getNodeLabels(nodes []corev1.Node, nodeName string) map[string]string {
+	for _, node := range nodes {
+		if node.Name == nodeName {
+			return node.Labels
 		}
 	}
-	return ret
-
+	return nil
 }