Skip to content

Commit

Permalink
test: implement e2e test of the deny-label-ns flag
Browse files Browse the repository at this point in the history 
Signed-off-by: AhmedGrati <[email protected]>
  • Loading branch information
@TessaIO
TessaIO committed Mar 6, 2023
1 parent 163a6dc commit ba1ad9e
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 27 deletions.
12 changes: 12 additions & 0 deletions test/e2e/data/nodefeature-3.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: nfd.k8s-sigs.io/v1alpha1
kind: NodeFeature
metadata:
name: zzz-e2e-features-3
spec:
labels:
random.denied.ns/e2e-nodefeature-test-1: "denied-ns"
random.unwanted.ns/e2e-nodefeature-test-2: "unwanted-ns"
custom.vendor.io/e2e-nodefeature-test-3: "vendor-ns"
kubernetes.io/denied-label: "kubernetes-ns-1"
subns.kubernetes.io/denied-label: "kubernetes-ns-2"
e2e-nodefeature-test-4: "obj-4"
130 changes: 103 additions & 27 deletions test/e2e/node_feature_discovery_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,10 @@ var _ = SIGDescribe("NFD master and worker", func() {
testpod.SpecWithContainerImage(dockerImage()),
testpod.SpecWithTolerations(testTolerations),
testpod.SpecWithContainerExtraArgs("-enable-taints"),
testpod.SpecWithContainerExtraArgs(
"-deny-label-ns=*.denied.ns,random.unwanted.ns,*.vendor.io",
"-extra-label-ns=custom.vendor.io",
),
)
masterPod := e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(podSpecOpts...))

Expand Down Expand Up @@ -518,14 +522,17 @@ var _ = SIGDescribe("NFD master and worker", func() {
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "overridden",
},
}
Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())

Expect(checkForNodeLabels(f.ClientSet,
expectedLabels, nodes,
)).NotTo(HaveOccurred())
By("Deleting NodeFeature object")
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(context.TODO(), nodeFeatures[0], metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())

By("Verifying node labels from NodeFeature object were removed")
Expect(waitForNfdNodeLabels(f.ClientSet, nil)).NotTo(HaveOccurred())
Expect(checkForNodeLabels(f.ClientSet,
nil, nodes,
)).NotTo(HaveOccurred())

By("Creating nfd-worker daemonset")
podSpecOpts := createPodSpecOpts(
Expand All @@ -547,7 +554,9 @@ var _ = SIGDescribe("NFD master and worker", func() {
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "true",
},
}
Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
Expect(checkForNodeLabels(f.ClientSet,
expectedLabels, nodes,
)).NotTo(HaveOccurred())

By("Re-creating NodeFeature object")
_, err = testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
Expand All @@ -561,7 +570,9 @@ var _ = SIGDescribe("NFD master and worker", func() {
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true",
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "overridden",
}
Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
Expect(checkForNodeLabels(f.ClientSet,
expectedLabels, nodes,
)).NotTo(HaveOccurred())

By("Creating extra namespace")
extraNs, err := f.CreateNamespace("node-feature-discvery-extra-ns", nil)
Expand All @@ -574,25 +585,73 @@ var _ = SIGDescribe("NFD master and worker", func() {
By("Verifying node labels from NodeFeature object #2 are created")
expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-1"] = "overridden-from-obj-2"
expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-3"] = "obj-2"
Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred())
Expect(checkForNodeLabels(f.ClientSet,
expectedLabels,
nodes,
)).NotTo(HaveOccurred())
})

It("denied labels should not be created by the NodeFeature object", func() {
if !useNodeFeatureApi {
Skip("NodeFeature API not enabled")
}

nodes, err := getNonControlPlaneNodes(f.ClientSet)
Expect(err).NotTo(HaveOccurred())

targetNodeName := nodes[0].Name
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")

// Apply Node Feature object
By("Create NodeFeature object")
nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-3.yaml", f.Namespace.Name, targetNodeName)
Expect(err).NotTo(HaveOccurred())

// Verify that denied label was not added
By("Verifying that denied labels were not added")
expectedLabels := map[string]k8sLabels{
targetNodeName: {
nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-4": "obj-4",
"custom.vendor.io/e2e-nodefeature-test-3": "vendor-ns",
},
}
Expect(checkForNodeLabels(
f.ClientSet,
expectedLabels,
nodes,
)).NotTo(HaveOccurred())

By("Deleting NodeFeature object")
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(context.TODO(), nodeFeatures[0], metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())

Expect(checkForNodeLabels(
f.ClientSet,
nil,
nodes,
)).NotTo(HaveOccurred())
})
})

//
// Test NodeFeatureRule
//
Context("and nfd-worker and NodeFeatureRules objects deployed", func() {
It("custom labels from the NodeFeatureRule rules should be created", func() {
nodes, err := getNonControlPlaneNodes(f.ClientSet)
Expect(err).NotTo(HaveOccurred())

targetNodeName := nodes[0].Name
Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found")

By("Creating nfd-worker config")
cm := testutils.NewConfigMap("nfd-worker-conf", "nfd-worker.conf", `
core:
sleepInterval: "1s"
featureSources: ["fake"]
labelSources: []
`)
cm, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm, metav1.CreateOptions{})
cm, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())

By("Creating nfd-worker daemonset")
podSpecOpts := createPodSpecOpts(
testpod.SpecWithContainerImage(dockerImage()),
Expand All @@ -618,7 +677,11 @@ core:
Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred())

By("Verifying node labels from NodeFeatureRules #1")
Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred())
Expect(checkForNodeLabels(
f.ClientSet,
expected,
nodes,
)).NotTo(HaveOccurred())

By("Creating NodeFeatureRules #2")
Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred())
Expand All @@ -629,7 +692,11 @@ core:
expected["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_2"] = "found"

By("Verifying node labels from NodeFeatureRules #1 and #2")
Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred())
Expect(checkForNodeLabels(
f.ClientSet,
expected,
nodes,
)).NotTo(HaveOccurred())

// Add features from NodeFeatureRule #3
By("Creating NodeFeatureRules #3")
Expand Down Expand Up @@ -686,7 +753,6 @@ core:
// Run the actual tests
Context("when running NFD with gRPC API enabled", func() {
nfdTestSuite(false)

})

Context("when running NFD with NodeFeature CRD API enabled", func() {
Expand Down Expand Up @@ -725,24 +791,27 @@ func waitForNfdNodeAnnotations(cli clientset.Interface, expected map[string]stri

type k8sLabels map[string]string

// waitForNfdNodeLabels waits for node to be labeled as expected.
func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]k8sLabels) error {
// checkForNfdNodeLabels waits and checks that node is labeled as expected.
func checkForNodeLabels(cli clientset.Interface, expectedNewLabels map[string]k8sLabels, oldNodes []corev1.Node) error {

poll := func() error {
nodes, err := getNonControlPlaneNodes(cli)
if err != nil {
return err
}
for _, node := range nodes {
labels := nfdLabels(node.Labels)
nodeExpected, ok := expected[node.Name]
nodeExpected, ok := expectedNewLabels[node.Name]
if !ok {
nodeExpected = k8sLabels{}
if defaultExpected, ok := expected["*"]; ok {
if defaultExpected, ok := expectedNewLabels["*"]; ok {
nodeExpected = defaultExpected
}
}
if !cmp.Equal(nodeExpected, labels) {
return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(nodeExpected, labels))

oldLabels := getNodeLabels(oldNodes, node.Name)
newNodeLabels := mergeMaps(oldLabels, nodeExpected)
if !cmp.Equal(node.Labels, newNodeLabels) {
return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(nodeExpected, node.Labels))
}
}
return nil
Expand Down Expand Up @@ -828,15 +897,22 @@ func getNonControlPlaneNodes(cli clientset.Interface) ([]corev1.Node, error) {
return out, nil
}

// nfdLabels gets labels that are in the nfd label namespace.
func nfdLabels(labels map[string]string) k8sLabels {
ret := k8sLabels{}

for key, val := range labels {
if strings.HasPrefix(key, nfdv1alpha1.FeatureLabelNs) {
ret[key] = val
func getNodeLabels(nodes []corev1.Node, nodeName string) map[string]string {
for _, node := range nodes {
if node.Name == nodeName {
return node.Labels
}
}
return ret
return nil
}

func mergeMaps(a map[string]string, b map[string]string) map[string]string {
result := map[string]string{}
for k, v := range a {
result[k] = v
}
for k, v := range b {
result[k] = v
}
return result
}

0 comments on commit ba1ad9e

Please sign in to comment.