diff --git a/test/e2e/data/nodefeaturerule-1.yaml b/test/e2e/data/nodefeaturerule-1.yaml new file mode 100644 index 0000000000..ea1f2f2fd1 --- /dev/null +++ b/test/e2e/data/nodefeaturerule-1.yaml @@ -0,0 +1,76 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: e2e-test-1 +spec: + rules: + # + # Simple test rules for flag features + # + - name: "e2e-flag-test-1" + labels: + e2e-flag-test-1: "true" + vars: + e2e-flag-test-1.not: "false" + matchFeatures: + - feature: "fake.flag" + matchExpressions: + "flag_1": {op: Exists} + + # Negative test not supposed to create a label + - name: "e2e-flag-test-neg-1" + labels: + e2e-flag-test-neg-1: "true" + matchFeatures: + - feature: "fake.flag" + matchExpressions: + "flag_1": {op: DoesNotExist} + + # + # Simple test rules for attribute features + # + - name: "e2e-attribute-test-1" + labels: + e2e-attribute-test-1: "true" + vars: + e2e-attribute-test-1.not: "false" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsFalse} + + # Negative test not supposed to create a label + - name: "e2e-attribute-test-neg-1" + labels: + e2e-attribute-test-neg-1: "true" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsTrue} + + # + # Simple test rules for instnace features + # + - name: "e2e-instance-test-1" + labels: + e2e-instance-test-1: "true" + vars: + e2e-instance-test-1.not: "false" + e2e-instance-test-1.123: "123" + matchFeatures: + - feature: "fake.instance" + matchExpressions: + "attr_1": {op: In, value: ["true"]} + "attr_3": {op: Gt, value: ["10"]} + + # Negative test not supposed to create a label + - name: "e2e-instance-test-neg-1" + labels: + e2e-instance-test-neg-1: "true" + matchFeatures: + - feature: "fake.instance" + matchExpressions: + "attr_1": {op: In, value: ["true"]} + "attr_3": {op: Lt, value: ["10"]} diff --git a/test/e2e/data/nodefeaturerule-2.yaml b/test/e2e/data/nodefeaturerule-2.yaml new file mode 100644 index 0000000000..c067bba762 --- /dev/null +++ b/test/e2e/data/nodefeaturerule-2.yaml @@ -0,0 +1,41 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: e2e-test-2 +spec: + rules: + # + # More complex rule testing backreferencing and matchAny field + # + - name: "e2e-matchany-test-1" + labels: + e2e-matchany-test-1: "true" + vars: + e2e-instance-test-1.not: "false" + matchFeatures: + - feature: "rule.matched" + matchExpressions: + "e2e-attribute-test-1": {op: InRegexp, value: ["^tru"]} + "e2e-instance-test-1.123": {op: In, value: ["1", "12", "123"]} + matchAny: + - matchFeatures: + - feature: "fake.instance" + matchExpressions: + "attr_1": {op: In, value: ["nomatch"]} + - matchFeatures: + - feature: "fake.instance" + matchExpressions: + "attr_3": {op: In, value: ["100"]} + + # + # Simple test for templating + # + - name: "e2e-template-test-1" + labelsTemplate: | + {{ range .fake.instance }}e2e-template-test-1-{{ .name }}=found + {{ end }} + matchFeatures: + - feature: "fake.instance" + matchExpressions: + "attr_1": {op: In, value: ["true"]} + diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 744ec3ba15..36044b9d0d 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -24,10 +24,13 @@ import ( "strings" "time" + "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" @@ -35,6 +38,7 @@ import ( e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" master "sigs.k8s.io/node-feature-discovery/pkg/nfd-master" "sigs.k8s.io/node-feature-discovery/source/custom" @@ -419,6 +423,103 @@ var _ = SIGDescribe("Node Feature Discovery", func() { }) }) - }) + // + // Test NodeFeatureRule + // + Context("and nfd-worker and NodeFeatureRules objects deployed", func() { + var extClient *extclient.Clientset + var nfdClient *nfdclient.Clientset + var crd *apiextensionsv1.CustomResourceDefinition + + BeforeEach(func() { + // Create clients for apiextensions and our CRD api + extClient = extclient.NewForConfigOrDie(f.ClientConfig()) + nfdClient = nfdclient.NewForConfigOrDie(f.ClientConfig()) + + // Create CRDs + By("Creating NodeFeatureRule CRD") + var err error + crd, err = testutils.CreateNodeFeatureRulesCRD(extClient) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("custom labels from the NodeFeatureRule rules should be created", func() { + By("Creating nfd-worker daemonset") + workerArgs := []string{"-feature-sources=fake", "-label-sources=", "-sleep-interval=1s"} + workerDS := testutils.NFDWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), workerArgs) + workerDS, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for daemonset pods to be ready") + Expect(testutils.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + + expected := map[string]string{ + "feature.node.kubernetes.io/e2e-flag-test-1": "true", + "feature.node.kubernetes.io/e2e-attribute-test-1": "true", + "feature.node.kubernetes.io/e2e-instance-test-1": "true"} + + By("Creating NodeFeatureRules #1") + Expect(testutils.CreateNodeFeatureRuleFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeatureRules #1") + Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) + + By("Creating NodeFeatureRules #2") + Expect(testutils.CreateNodeFeatureRuleFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred()) + + // Add features from NodeFeatureRule #2 + expected["feature.node.kubernetes.io/e2e-matchany-test-1"] = "true" + expected["feature.node.kubernetes.io/e2e-template-test-1-instance_1"] = "found" + expected["feature.node.kubernetes.io/e2e-template-test-1-instance_2"] = "found" + By("Verifying node labels from NodeFeatureRules #1 and #2") + Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) + }) + }) + }) }) + +// waitForNfdNodeLabels waits for node to be labeled as expected. +func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) error { + poll := func() error { + nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return err + } + for _, node := range nodeList.Items { + labels := nfdLabels(node.Labels) + if !cmp.Equal(expected, labels) { + return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expected, labels)) + } + } + return nil + } + + // Simple and stupid re-try loop + var err error + for retry := 0; retry < 3; retry++ { + if err = poll(); err == nil { + return nil + } + time.Sleep(2 * time.Second) + } + return err +} + +// nfdLabels gets labels that are in the nfd label namespace. +func nfdLabels(labels map[string]string) map[string]string { + ret := map[string]string{} + + for key, val := range labels { + if strings.HasPrefix(key, master.FeatureLabelNs) { + ret[key] = val + } + } + return ret + +} diff --git a/test/e2e/utils/crd.go b/test/e2e/utils/crd.go new file mode 100644 index 0000000000..5a45e642db --- /dev/null +++ b/test/e2e/utils/crd.go @@ -0,0 +1,110 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" + nfdclientset "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" + nfdscheme "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned/scheme" +) + +var packagePath string + +// CreateNodeFeatureRulesCRD creates the NodeFeatureRule CRD in the API server. +func CreateNodeFeatureRulesCRD(cli extclient.Interface) (*apiextensionsv1.CustomResourceDefinition, error) { + crd, err := crdFromFile(filepath.Join(packagePath, "..", "..", "..", "deployment", "base", "nfd-crds", "nodefeaturerule-crd.yaml")) + if err != nil { + return nil, err + } + + // Delete existing CRD (if any) with this we also get rid of stale objects + err = cli.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return nil, fmt.Errorf("failed to delete NodeFeatureRule CRD: %w", err) + } + + return cli.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}) +} + +// CreateNodeFeatureRuleFromFile creates a NodeFeatureRule object from a given file located under test data directory. +func CreateNodeFeatureRuleFromFile(cli nfdclientset.Interface, filename string) error { + obj, err := nodeFeatureRuleFromFile(filepath.Join(packagePath, "..", "data", filename)) + if err != nil { + return err + } + _, err = cli.NfdV1alpha1().NodeFeatureRules().Create(context.TODO(), obj, metav1.CreateOptions{}) + return err +} + +func apiObjFromFile(path string, decoder apiruntime.Decoder) (apiruntime.Object, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + obj, _, err := decoder.Decode(data, nil, nil) + return obj, err +} + +// crdFromFile creates a CustomResourceDefinition API object from a file. +func crdFromFile(path string) (*apiextensionsv1.CustomResourceDefinition, error) { + obj, err := apiObjFromFile(path, scheme.Codecs.UniversalDeserializer()) + if err != nil { + return nil, err + } + + crd, ok := obj.(*apiextensionsv1.CustomResourceDefinition) + if !ok { + return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path) + } + + return crd, nil +} + +func nodeFeatureRuleFromFile(path string) (*nfdv1alpha1.NodeFeatureRule, error) { + obj, err := apiObjFromFile(path, nfdscheme.Codecs.UniversalDeserializer()) + if err != nil { + return nil, err + } + + crd, ok := obj.(*nfdv1alpha1.NodeFeatureRule) + if !ok { + return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path) + } + + return crd, nil +} + +func init() { + _, thisFile, _, _ := runtime.Caller(0) + packagePath = filepath.Dir(thisFile) + + // Register k8s scheme to be able to create CRDs + _ = apiextensionsv1.AddToScheme(scheme.Scheme) +} diff --git a/test/e2e/utils/rbac.go b/test/e2e/utils/rbac.go index c5b93b0cd5..655c00493b 100644 --- a/test/e2e/utils/rbac.go +++ b/test/e2e/utils/rbac.go @@ -128,6 +128,11 @@ func createClusterRoleMaster(cs clientset.Interface) (*rbacv1.ClusterRole, error Resources: []string{"nodes"}, Verbs: []string{"get", "patch", "update"}, }, + { + APIGroups: []string{"nfd.k8s-sigs.io"}, + Resources: []string{"nodefeaturerules"}, + Verbs: []string{"get", "list", "watch"}, + }, { APIGroups: []string{"topology.node.k8s.io"}, Resources: []string{"noderesourcetopologies"},