From 6ef6112d58519d2d4ed663c1b4a4de3353acf5d6 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 7 Jul 2022 10:40:19 +0300 Subject: [PATCH] test/e2e: add tests for NodeFeatureRules Add tests covering the basic functionality of NodeFeatureRule objects, covering different feature types ("flag features", "attribute features" and "instance features") as well as backreferencing (using the output of previously run rules) and templating. The test relies on the "fake" feature source and its default configuration. --- test/e2e/data/nodefeaturerule-1.yaml | 76 ++++++++++++++++++ test/e2e/data/nodefeaturerule-2.yaml | 41 ++++++++++ test/e2e/node_feature_discovery.go | 103 ++++++++++++++++++++++++- test/e2e/utils/crd.go | 110 +++++++++++++++++++++++++++ test/e2e/utils/rbac.go | 5 ++ 5 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 test/e2e/data/nodefeaturerule-1.yaml create mode 100644 test/e2e/data/nodefeaturerule-2.yaml create mode 100644 test/e2e/utils/crd.go diff --git a/test/e2e/data/nodefeaturerule-1.yaml b/test/e2e/data/nodefeaturerule-1.yaml new file mode 100644 index 0000000000..ea1f2f2fd1 --- /dev/null +++ b/test/e2e/data/nodefeaturerule-1.yaml @@ -0,0 +1,76 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: e2e-test-1 +spec: + rules: + # + # Simple test rules for flag features + # + - name: "e2e-flag-test-1" + labels: + e2e-flag-test-1: "true" + vars: + e2e-flag-test-1.not: "false" + matchFeatures: + - feature: "fake.flag" + matchExpressions: + "flag_1": {op: Exists} + + # Negative test not supposed to create a label + - name: "e2e-flag-test-neg-1" + labels: + e2e-flag-test-neg-1: "true" + matchFeatures: + - feature: "fake.flag" + matchExpressions: + "flag_1": {op: DoesNotExist} + + # + # Simple test rules for attribute features + # + - name: "e2e-attribute-test-1" + labels: + e2e-attribute-test-1: "true" + vars: + e2e-attribute-test-1.not: "false" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsFalse} + + # Negative test not supposed to create a label + - name: "e2e-attribute-test-neg-1" + labels: + e2e-attribute-test-neg-1: "true" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsTrue} + + # + # Simple test rules for instnace features + # + - name: "e2e-instance-test-1" + labels: + e2e-instance-test-1: "true" + vars: + e2e-instance-test-1.not: "false" + e2e-instance-test-1.123: "123" + matchFeatures: + - feature: "fake.instance" + matchExpressions: + "attr_1": {op: In, value: ["true"]} + "attr_3": {op: Gt, value: ["10"]} + + # Negative test not supposed to create a label + - name: "e2e-instance-test-neg-1" + labels: + e2e-instance-test-neg-1: "true" + matchFeatures: + - feature: "fake.instance" + matchExpressions: + "attr_1": {op: In, value: ["true"]} + "attr_3": {op: Lt, value: ["10"]} diff --git a/test/e2e/data/nodefeaturerule-2.yaml b/test/e2e/data/nodefeaturerule-2.yaml new file mode 100644 index 0000000000..c067bba762 --- /dev/null +++ b/test/e2e/data/nodefeaturerule-2.yaml @@ -0,0 +1,41 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: e2e-test-2 +spec: + rules: + # + # More complex rule testing backreferencing and matchAny field + # + - name: "e2e-matchany-test-1" + labels: + e2e-matchany-test-1: "true" + vars: + e2e-instance-test-1.not: "false" + matchFeatures: + - feature: "rule.matched" + matchExpressions: + "e2e-attribute-test-1": {op: InRegexp, value: ["^tru"]} + "e2e-instance-test-1.123": {op: In, value: ["1", "12", "123"]} + matchAny: + - matchFeatures: + - feature: "fake.instance" + matchExpressions: + "attr_1": {op: In, value: ["nomatch"]} + - matchFeatures: + - feature: "fake.instance" + matchExpressions: + "attr_3": {op: In, value: ["100"]} + + # + # Simple test for templating + # + - name: "e2e-template-test-1" + labelsTemplate: | + {{ range .fake.instance }}e2e-template-test-1-{{ .name }}=found + {{ end }} + matchFeatures: + - feature: "fake.instance" + matchExpressions: + "attr_1": {op: In, value: ["true"]} + diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 41cbc59eb7..9a1c1e8aba 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -24,10 +24,13 @@ import ( "strings" "time" + "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" @@ -35,6 +38,7 @@ import ( e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" "sigs.k8s.io/node-feature-discovery/source/custom" @@ -419,6 +423,103 @@ var _ = SIGDescribe("Node Feature Discovery", func() { }) }) - }) + // + // Test NodeFeatureRule + // + Context("and nfd-worker and NodeFeatureRules objects deployed", func() { + var extClient *extclient.Clientset + var nfdClient *nfdclient.Clientset + var crd *apiextensionsv1.CustomResourceDefinition + + BeforeEach(func() { + // Create clients for apiextensions and our CRD api + extClient = extclient.NewForConfigOrDie(f.ClientConfig()) + nfdClient = nfdclient.NewForConfigOrDie(f.ClientConfig()) + + // Create CRDs + By("Creating NodeFeatureRule CRD") + var err error + crd, err = testutils.CreateNodeFeatureRulesCRD(extClient) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("custom labels from the NodeFeatureRule rules should be created", func() { + By("Creating nfd-worker daemonset") + workerArgs := []string{"-feature-sources=fake", "-label-sources=", "-sleep-interval=1s"} + workerDS := testutils.NFDWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), workerArgs) + workerDS, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for daemonset pods to be ready") + Expect(testutils.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + + expected := map[string]string{ + "feature.node.kubernetes.io/e2e-flag-test-1": "true", + "feature.node.kubernetes.io/e2e-attribute-test-1": "true", + "feature.node.kubernetes.io/e2e-instance-test-1": "true"} + + By("Creating NodeFeatureRules #1") + Expect(testutils.CreateNodeFeatureRuleFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeatureRules #1") + Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) + + By("Creating NodeFeatureRules #2") + Expect(testutils.CreateNodeFeatureRuleFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred()) + + // Add features from NodeFeatureRule #2 + expected["feature.node.kubernetes.io/e2e-matchany-test-1"] = "true" + expected["feature.node.kubernetes.io/e2e-template-test-1-instance_1"] = "found" + expected["feature.node.kubernetes.io/e2e-template-test-1-instance_2"] = "found" + By("Verifying node labels from NodeFeatureRules #1 and #2") + Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) + }) + }) + }) }) + +// waitForNfdNodeLabels waits for node to be labeled as expected. +func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) error { + poll := func() error { + nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return err + } + for _, node := range nodeList.Items { + labels := nfdLabels(node.Labels) + if !cmp.Equal(expected, labels) { + return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expected, labels)) + } + } + return nil + } + + // Simple and stupid re-try loop + var err error + for retry := 0; retry < 3; retry++ { + if err = poll(); err == nil { + return nil + } + time.Sleep(2 * time.Second) + } + return err +} + +// nfdLabels gets labels that are in the nfd label namespace. +func nfdLabels(labels map[string]string) map[string]string { + ret := map[string]string{} + + for key, val := range labels { + if strings.HasPrefix(key, nfdv1alpha1.FeatureLabelNs) { + ret[key] = val + } + } + return ret + +} diff --git a/test/e2e/utils/crd.go b/test/e2e/utils/crd.go new file mode 100644 index 0000000000..5a45e642db --- /dev/null +++ b/test/e2e/utils/crd.go @@ -0,0 +1,110 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" + nfdclientset "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" + nfdscheme "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned/scheme" +) + +var packagePath string + +// CreateNodeFeatureRulesCRD creates the NodeFeatureRule CRD in the API server. +func CreateNodeFeatureRulesCRD(cli extclient.Interface) (*apiextensionsv1.CustomResourceDefinition, error) { + crd, err := crdFromFile(filepath.Join(packagePath, "..", "..", "..", "deployment", "base", "nfd-crds", "nodefeaturerule-crd.yaml")) + if err != nil { + return nil, err + } + + // Delete existing CRD (if any) with this we also get rid of stale objects + err = cli.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return nil, fmt.Errorf("failed to delete NodeFeatureRule CRD: %w", err) + } + + return cli.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}) +} + +// CreateNodeFeatureRuleFromFile creates a NodeFeatureRule object from a given file located under test data directory. +func CreateNodeFeatureRuleFromFile(cli nfdclientset.Interface, filename string) error { + obj, err := nodeFeatureRuleFromFile(filepath.Join(packagePath, "..", "data", filename)) + if err != nil { + return err + } + _, err = cli.NfdV1alpha1().NodeFeatureRules().Create(context.TODO(), obj, metav1.CreateOptions{}) + return err +} + +func apiObjFromFile(path string, decoder apiruntime.Decoder) (apiruntime.Object, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + obj, _, err := decoder.Decode(data, nil, nil) + return obj, err +} + +// crdFromFile creates a CustomResourceDefinition API object from a file. +func crdFromFile(path string) (*apiextensionsv1.CustomResourceDefinition, error) { + obj, err := apiObjFromFile(path, scheme.Codecs.UniversalDeserializer()) + if err != nil { + return nil, err + } + + crd, ok := obj.(*apiextensionsv1.CustomResourceDefinition) + if !ok { + return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path) + } + + return crd, nil +} + +func nodeFeatureRuleFromFile(path string) (*nfdv1alpha1.NodeFeatureRule, error) { + obj, err := apiObjFromFile(path, nfdscheme.Codecs.UniversalDeserializer()) + if err != nil { + return nil, err + } + + crd, ok := obj.(*nfdv1alpha1.NodeFeatureRule) + if !ok { + return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path) + } + + return crd, nil +} + +func init() { + _, thisFile, _, _ := runtime.Caller(0) + packagePath = filepath.Dir(thisFile) + + // Register k8s scheme to be able to create CRDs + _ = apiextensionsv1.AddToScheme(scheme.Scheme) +} diff --git a/test/e2e/utils/rbac.go b/test/e2e/utils/rbac.go index c5b93b0cd5..655c00493b 100644 --- a/test/e2e/utils/rbac.go +++ b/test/e2e/utils/rbac.go @@ -128,6 +128,11 @@ func createClusterRoleMaster(cs clientset.Interface) (*rbacv1.ClusterRole, error Resources: []string{"nodes"}, Verbs: []string{"get", "patch", "update"}, }, + { + APIGroups: []string{"nfd.k8s-sigs.io"}, + Resources: []string{"nodefeaturerules"}, + Verbs: []string{"get", "list", "watch"}, + }, { APIGroups: []string{"topology.node.k8s.io"}, Resources: []string{"noderesourcetopologies"},