Skip to content

Commit

Permalink
Merge pull request #1426 from marquiz/devel/e2e-annotations-validation
Browse files Browse the repository at this point in the history 
test/e2e: stricter validation of node annotations
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Oct 23, 2023
2 parents 6144f3d + ddec2e2 commit 29ee4fe
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 47 deletions.
28 changes: 10 additions & 18 deletions test/e2e/gomega.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,9 @@ func eventuallyNonControlPlaneNodes(ctx context.Context, cli clientset.Interface

// MatchLabels returns a specialized Gomega matcher for checking if a list of
// nodes are labeled as expected.
func MatchLabels(expectedNew map[string]k8sLabels, oldNodes []corev1.Node, ignoreUnexpected bool) gomegatypes.GomegaMatcher {
func MatchLabels(expectedNew map[string]k8sLabels, oldNodes []corev1.Node) gomegatypes.GomegaMatcher {
matcher := &nodeIterablePropertyMatcher[k8sLabels]{
propertyName: "labels",
ignoreUnexpected: ignoreUnexpected,
propertyName: "labels",
matchFunc: func(newNode, oldNode corev1.Node, expected k8sLabels) ([]string, []string, []string) {
expectedAll := maps.Clone(oldNode.Labels)
maps.Copy(expectedAll, expected)
Expand All @@ -64,10 +63,9 @@ func MatchLabels(expectedNew map[string]k8sLabels, oldNodes []corev1.Node, ignor

// MatchAnnotations returns a specialized Gomega matcher for checking if a list of
// nodes are annotated as expected.
func MatchAnnotations(expectedNew map[string]k8sAnnotations, oldNodes []corev1.Node, ignoreUnexpected bool) gomegatypes.GomegaMatcher {
func MatchAnnotations(expectedNew map[string]k8sAnnotations, oldNodes []corev1.Node) gomegatypes.GomegaMatcher {
matcher := &nodeIterablePropertyMatcher[k8sAnnotations]{
propertyName: "annotations",
ignoreUnexpected: ignoreUnexpected,
propertyName: "annotations",
matchFunc: func(newNode, oldNode corev1.Node, expected k8sAnnotations) ([]string, []string, []string) {
expectedAll := maps.Clone(oldNode.Annotations)
maps.Copy(expectedAll, expected)
Expand All @@ -84,10 +82,9 @@ func MatchAnnotations(expectedNew map[string]k8sAnnotations, oldNodes []corev1.N

// MatchCapacity returns a specialized Gomega matcher for checking if a list of
// nodes have resource capacity as expected.
func MatchCapacity(expectedNew map[string]corev1.ResourceList, oldNodes []corev1.Node, ignoreUnexpected bool) gomegatypes.GomegaMatcher {
func MatchCapacity(expectedNew map[string]corev1.ResourceList, oldNodes []corev1.Node) gomegatypes.GomegaMatcher {
matcher := &nodeIterablePropertyMatcher[corev1.ResourceList]{
propertyName: "resource capacity",
ignoreUnexpected: ignoreUnexpected,
propertyName: "resource capacity",
matchFunc: func(newNode, oldNode corev1.Node, expected corev1.ResourceList) ([]string, []string, []string) {
expectedAll := oldNode.Status.DeepCopy().Capacity
maps.Copy(expectedAll, expected)
Expand All @@ -104,10 +101,9 @@ func MatchCapacity(expectedNew map[string]corev1.ResourceList, oldNodes []corev1

// MatchTaints returns a specialized Gomega matcher for checking if a list of
// nodes are tainted as expected.
func MatchTaints(expectedNew map[string][]corev1.Taint, oldNodes []corev1.Node, ignoreUnexpected bool) gomegatypes.GomegaMatcher {
func MatchTaints(expectedNew map[string][]corev1.Taint, oldNodes []corev1.Node) gomegatypes.GomegaMatcher {
matcher := &nodeIterablePropertyMatcher[[]corev1.Taint]{
propertyName: "taints",
ignoreUnexpected: ignoreUnexpected,
propertyName: "taints",
matchFunc: func(newNode, oldNode corev1.Node, expected []corev1.Taint) (missing, invalid, unexpected []string) {
expectedAll := oldNode.Spec.DeepCopy().Taints
expectedAll = append(expectedAll, expected...)
Expand Down Expand Up @@ -203,9 +199,8 @@ func (m *nodeListPropertyMatcher[T]) NegatedFailureMessage(actual interface{}) s
// nodeIterablePropertyMatcher is a nodePropertyMatcher for matching iterable
// elements such as maps or lists.
type nodeIterablePropertyMatcher[T any] struct {
propertyName string
ignoreUnexpected bool
matchFunc func(newNode, oldNode corev1.Node, expected T) ([]string, []string, []string)
propertyName string
matchFunc func(newNode, oldNode corev1.Node, expected T) ([]string, []string, []string)

// TODO remove nolint when golangci-lint is able to cope with generics
node *corev1.Node //nolint:unused
Expand All @@ -222,9 +217,6 @@ func (m *nodeIterablePropertyMatcher[T]) match(newNode, oldNode corev1.Node, exp
m.node = &newNode
m.missing, m.invalidValue, m.unexpected = m.matchFunc(newNode, oldNode, expected)

if m.ignoreUnexpected {
m.unexpected = nil
}
return len(m.missing) == 0 && len(m.invalidValue) == 0 && len(m.unexpected) == 0
}

Expand Down
65 changes: 36 additions & 29 deletions test/e2e/node_feature_discovery_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ var _ = SIGDescribe("NFD master and worker", func() {
},
"*": {},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))

checkNodeFeatureObject(ctx, workerPod.Spec.NodeName)

Expand Down Expand Up @@ -478,7 +478,7 @@ var _ = SIGDescribe("NFD master and worker", func() {
},
"*": {},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))

By("Deleting nfd-worker daemonset")
err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(ctx, workerDS.Name, metav1.DeleteOptions{})
Expand Down Expand Up @@ -523,14 +523,14 @@ var _ = SIGDescribe("NFD master and worker", func() {
},
"*": {},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
By("Deleting NodeFeature object")
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())

By("Verifying node labels from NodeFeature object were removed")
expectedLabels[targetNodeName] = k8sLabels{}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))

By("Creating nfd-worker daemonset")
podSpecOpts := createPodSpecOpts(
Expand All @@ -552,7 +552,7 @@ var _ = SIGDescribe("NFD master and worker", func() {
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "true",
},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))

By("Re-creating NodeFeature object")
_, err = testutils.CreateOrUpdateNodeFeaturesFromFile(ctx, nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName)
Expand All @@ -566,7 +566,7 @@ var _ = SIGDescribe("NFD master and worker", func() {
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true",
nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "overridden",
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))

By("Creating extra namespace")
extraNs, err := f.CreateNamespace(ctx, "node-feature-discvery-extra-ns", nil)
Expand All @@ -579,7 +579,7 @@ var _ = SIGDescribe("NFD master and worker", func() {
By("Verifying node labels from NodeFeature object #2 are created")
expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-1"] = "overridden-from-obj-2"
expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-3"] = "obj-2"
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))

By("Deleting NodeFeature object from the extra namespace")
err = nfdClient.NfdV1alpha1().NodeFeatures(extraNs.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
Expand All @@ -588,7 +588,7 @@ var _ = SIGDescribe("NFD master and worker", func() {
By("Verifying node labels from NodeFeature object were removed")
expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-1"] = "obj-1"
delete(expectedLabels[targetNodeName], nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-3")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
})

It("denied labels should not be created by the NodeFeature object", func(ctx context.Context) {
Expand All @@ -615,14 +615,14 @@ var _ = SIGDescribe("NFD master and worker", func() {
"custom.vendor.io/e2e-nodefeature-test-3": "vendor-ns",
},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))

By("Deleting NodeFeature object")
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())

expectedLabels[targetNodeName] = k8sLabels{}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
})
})

Expand Down Expand Up @@ -698,7 +698,7 @@ core:
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred())

By("Verifying node labels from NodeFeatureRules #1")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))

By("Creating NodeFeatureRules #2")
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred())
Expand All @@ -708,9 +708,14 @@ core:
expectedLabels["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_1"] = "found"
expectedLabels["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_2"] = "found"
expectedLabels["*"][nfdv1alpha1.FeatureLabelNs+"/dynamic-label"] = "true"
expectedAnnotations := map[string]k8sAnnotations{
"*": {
"nfd.node.kubernetes.io/feature-labels": "dynamic-label,e2e-attribute-test-1,e2e-flag-test-1,e2e-instance-test-1,e2e-matchany-test-1,e2e-template-test-1-instance_1,e2e-template-test-1-instance_2"},
}

By("Verifying node labels from NodeFeatureRules #1 and #2")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))

// Add features from NodeFeatureRule #3
By("Creating NodeFeatureRules #3")
Expand All @@ -736,12 +741,10 @@ core:
},
},
}
expectedAnnotations := map[string]k8sAnnotations{
"*": {
"nfd.node.kubernetes.io/taints": "feature.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,feature.node.kubernetes.io/fake-dedicated-node=true:NoExecute,feature.node.kubernetes.io/performance-optimized-node=true:NoExecute"},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes, true))
expectedAnnotations["*"]["nfd.node.kubernetes.io/taints"] = "feature.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,feature.node.kubernetes.io/fake-dedicated-node=true:NoExecute,feature.node.kubernetes.io/performance-optimized-node=true:NoExecute"

eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))

By("Re-applying NodeFeatureRules #3 with updated taints")
Expect(testutils.UpdateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-3-updated.yaml")).NotTo(HaveOccurred())
Expand All @@ -760,16 +763,18 @@ core:
expectedAnnotations["*"]["nfd.node.kubernetes.io/taints"] = "feature.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,feature.node.kubernetes.io/foo=true:NoExecute"

By("Verifying updated node taints and annotation from NodeFeatureRules #3")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes, true))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))

By("Deleting NodeFeatureRule object")
err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, "e2e-test-3", metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
expectedTaints["*"] = []corev1.Taint{}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes, false))
delete(expectedAnnotations["*"], "nfd.node.kubernetes.io/taints")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchTaints(expectedTaints, nodes))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))

expectedAnnotations["*"] = k8sAnnotations{"nfd.node.kubernetes.io/extended-resources": "nons,vendor.feature.node.kubernetes.io/static,vendor.io/dynamic"}
expectedAnnotations["*"]["nfd.node.kubernetes.io/extended-resources"] = "nons,vendor.feature.node.kubernetes.io/static,vendor.io/dynamic"

expectedCapacity := map[string]corev1.ResourceList{
"*": {
Expand All @@ -783,18 +788,20 @@ core:
Expect(testutils.CreateNodeFeatureRulesFromFile(ctx, nfdClient, "nodefeaturerule-4.yaml")).NotTo(HaveOccurred())

By("Verifying node annotations from NodeFeatureRules #4")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes, true))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))

By("Verfiying node status capacity from NodeFeatureRules #4")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchCapacity(expectedCapacity, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchCapacity(expectedCapacity, nodes))

By("Deleting NodeFeatureRule object")
err = nfdClient.NfdV1alpha1().NodeFeatureRules().Delete(ctx, "e2e-extened-resource-test", metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())

By("Verfiying node status capacity from NodeFeatureRules #4")
expectedCapacity = map[string]corev1.ResourceList{"*": {}}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchCapacity(expectedCapacity, nodes, false))
delete(expectedAnnotations["*"], "nfd.node.kubernetes.io/extended-resources")
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchCapacity(expectedCapacity, nodes))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchAnnotations(expectedAnnotations, nodes))

By("Deleting nfd-worker daemonset")
err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(ctx, workerDS.Name, metav1.DeleteOptions{})
Expand Down Expand Up @@ -838,7 +845,7 @@ denyLabelNs: ["*.denied.ns","random.unwanted.ns"]
"custom.vendor.io/e2e-nodefeature-test-3": "vendor-ns",
},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
By("Deleting NodeFeature object")
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
Expand All @@ -859,7 +866,7 @@ denyLabelNs: []
"random.unwanted.ns/e2e-nodefeature-test-2": "unwanted-ns",
},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))
})
})

Expand Down Expand Up @@ -900,7 +907,7 @@ resyncPeriod: "1s"
},
"*": {},
}
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))

patches, err := json.Marshal(
[]apihelper.JsonPatch{
Expand All @@ -917,7 +924,7 @@ resyncPeriod: "1s"
_, err = f.ClientSet.CoreV1().Nodes().Patch(ctx, targetNodeName, types.JSONPatchType, patches, metav1.PatchOptions{})
Expect(err).NotTo(HaveOccurred())

eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes, false))
eventuallyNonControlPlaneNodes(ctx, f.ClientSet).Should(MatchLabels(expectedLabels, nodes))

By("Deleting NodeFeature object")
err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(ctx, nodeFeatures[0], metav1.DeleteOptions{})
Expand Down

0 comments on commit 29ee4fe

Please sign in to comment.