From d0d75838ad4fd12b698ee00c718a59c37dfe2e30 Mon Sep 17 00:00:00 2001 From: Artyom Lukianov Date: Mon, 23 Nov 2020 18:37:42 +0200 Subject: [PATCH] Move helpers method to the separate package Signed-off-by: Artyom Lukianov --- go.mod | 1 + test/e2e/node_feature_discovery.go | 416 ++--------------------------- test/e2e/utils/config.go | 104 ++++++++ test/e2e/utils/pod.go | 199 ++++++++++++++ test/e2e/utils/rbac.go | 138 ++++++++++ test/e2e/utils/service.go | 45 ++++ vendor/modules.txt | 1 + 7 files changed, 507 insertions(+), 397 deletions(-) create mode 100644 test/e2e/utils/config.go create mode 100644 test/e2e/utils/pod.go create mode 100644 test/e2e/utils/rbac.go create mode 100644 test/e2e/utils/service.go diff --git a/go.mod b/go.mod index 7881fd4ec7..3e22819bb5 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect google.golang.org/grpc v1.39.0 google.golang.org/protobuf v1.26.0-rc.1 + gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.21.2 diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 0b1d5cd53d..10a60eedcf 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -20,18 +20,14 @@ import ( "context" "flag" "fmt" - "io/ioutil" "path/filepath" - "regexp" "strings" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" @@ -42,375 +38,14 @@ import ( master "sigs.k8s.io/node-feature-discovery/pkg/nfd-master" "sigs.k8s.io/node-feature-discovery/source/custom" - "sigs.k8s.io/yaml" + testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils" ) var ( - dockerRepo = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from") - dockerTag = flag.String("nfd.tag", "master", "Docker tag to use") - e2eConfigFile = flag.String("nfd.e2e-config", "", "Configuration parameters for end-to-end tests") - openShift = flag.Bool("nfd.openshift", false, "Enable OpenShift specific bits") - pullIfNotPresent = flag.Bool("nfd.pull-if-not-present", false, "Pull Images if not present - not always") - - conf *e2eConfig + dockerRepo = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from") + dockerTag = flag.String("nfd.tag", "master", "Docker tag to use") ) -type e2eConfig struct { - DefaultFeatures *struct { - LabelWhitelist lookupMap - AnnotationWhitelist lookupMap - Nodes map[string]nodeConfig - } -} - -type nodeConfig struct { - nameRe *regexp.Regexp - ExpectedLabelValues map[string]string - ExpectedLabelKeys lookupMap - ExpectedAnnotationValues map[string]string - ExpectedAnnotationKeys lookupMap -} - -type lookupMap map[string]struct{} - -func (l *lookupMap) UnmarshalJSON(data []byte) error { - *l = lookupMap{} - slice := []string{} - - err := yaml.Unmarshal(data, &slice) - if err != nil { - return err - } - - for _, k := range slice { - (*l)[k] = struct{}{} - } - return nil -} - -func readConfig() { - // Read and parse only once - if conf != nil || *e2eConfigFile == "" { - return - } - - By("Reading end-to-end test configuration file") - data, err := ioutil.ReadFile(*e2eConfigFile) - Expect(err).NotTo(HaveOccurred()) - - By("Parsing end-to-end test configuration data") - err = yaml.Unmarshal(data, &conf) - Expect(err).NotTo(HaveOccurred()) - - // Pre-compile node name matching regexps - for name, nodeConf := range conf.DefaultFeatures.Nodes { - nodeConf.nameRe, err = regexp.Compile(name) - Expect(err).NotTo(HaveOccurred()) - conf.DefaultFeatures.Nodes[name] = nodeConf - } -} - -// Create required RBAC configuration -func configureRBAC(cs clientset.Interface, ns string) error { - _, err := createServiceAccount(cs, ns) - if err != nil { - return err - } - - _, err = createClusterRole(cs) - if err != nil { - return err - } - - _, err = createClusterRoleBinding(cs, ns) - if err != nil { - return err - } - - return nil -} - -// Remove RBAC configuration -func deconfigureRBAC(cs clientset.Interface, ns string) error { - err := cs.RbacV1().ClusterRoleBindings().Delete(context.TODO(), "nfd-master-e2e", metav1.DeleteOptions{}) - if err != nil { - return err - } - err = cs.RbacV1().ClusterRoles().Delete(context.TODO(), "nfd-master-e2e", metav1.DeleteOptions{}) - if err != nil { - return err - } - err = cs.CoreV1().ServiceAccounts(ns).Delete(context.TODO(), "nfd-master-e2e", metav1.DeleteOptions{}) - if err != nil { - return err - } - return nil -} - -// Configure service account required by NFD -func createServiceAccount(cs clientset.Interface, ns string) (*v1.ServiceAccount, error) { - sa := &v1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nfd-master-e2e", - Namespace: ns, - }, - } - return cs.CoreV1().ServiceAccounts(ns).Create(context.TODO(), sa, metav1.CreateOptions{}) -} - -// Configure cluster role required by NFD -func createClusterRole(cs clientset.Interface) (*rbacv1.ClusterRole, error) { - cr := &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nfd-master-e2e", - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "patch", "update"}, - }, - }, - } - if *openShift { - cr.Rules = append(cr.Rules, - rbacv1.PolicyRule{ - // needed on OpenShift clusters - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - ResourceNames: []string{"hostaccess"}, - Verbs: []string{"use"}, - }) - } - return cs.RbacV1().ClusterRoles().Update(context.TODO(), cr, metav1.UpdateOptions{}) -} - -// Configure cluster role binding required by NFD -func createClusterRoleBinding(cs clientset.Interface, ns string) (*rbacv1.ClusterRoleBinding, error) { - crb := &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nfd-master-e2e", - }, - Subjects: []rbacv1.Subject{ - { - Kind: rbacv1.ServiceAccountKind, - Name: "nfd-master-e2e", - Namespace: ns, - }, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: rbacv1.GroupName, - Kind: "ClusterRole", - Name: "nfd-master-e2e", - }, - } - - return cs.RbacV1().ClusterRoleBindings().Update(context.TODO(), crb, metav1.UpdateOptions{}) -} - -// createService creates nfd-master Service -func createService(cs clientset.Interface, ns string) (*v1.Service, error) { - svc := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nfd-master-e2e", - }, - Spec: v1.ServiceSpec{ - Selector: map[string]string{"name": "nfd-master-e2e"}, - Ports: []v1.ServicePort{ - { - Protocol: v1.ProtocolTCP, - Port: 8080, - }, - }, - Type: v1.ServiceTypeClusterIP, - }, - } - return cs.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{}) -} - -func nfdMasterPod(image string, onMasterNode bool) *v1.Pod { - p := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nfd-master-" + string(uuid.NewUUID()), - Labels: map[string]string{"name": "nfd-master-e2e"}, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "node-feature-discovery", - Image: image, - ImagePullPolicy: pullPolicy(), - Command: []string{"nfd-master"}, - Env: []v1.EnvVar{ - { - Name: "NODE_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - }, - }, - }, - ServiceAccountName: "nfd-master-e2e", - RestartPolicy: v1.RestartPolicyNever, - }, - } - if onMasterNode { - p.Spec.NodeSelector = map[string]string{"node-role.kubernetes.io/master": ""} - p.Spec.Tolerations = []v1.Toleration{ - { - Key: "node-role.kubernetes.io/master", - Operator: v1.TolerationOpEqual, - Value: "", - Effect: v1.TaintEffectNoSchedule, - }, - } - } - return p -} - -func nfdWorkerPod(image string, extraArgs []string) *v1.Pod { - p := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nfd-worker-" + string(uuid.NewUUID()), - }, - Spec: nfdWorkerPodSpec(image, extraArgs), - } - - p.Spec.RestartPolicy = v1.RestartPolicyNever - - return p -} - -func nfdWorkerDaemonSet(image string, extraArgs []string) *appsv1.DaemonSet { - return &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nfd-worker-" + string(uuid.NewUUID()), - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"name": "nfd-worker"}, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"name": "nfd-worker"}, - }, - Spec: nfdWorkerPodSpec(image, extraArgs), - }, - MinReadySeconds: 5, - }, - } -} - -func nfdWorkerPodSpec(image string, extraArgs []string) v1.PodSpec { - return v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "node-feature-discovery", - Image: image, - ImagePullPolicy: pullPolicy(), - Command: []string{"nfd-worker"}, - Args: append([]string{"--server=nfd-master-e2e:8080"}, extraArgs...), - Env: []v1.EnvVar{ - { - Name: "NODE_NAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "host-boot", - MountPath: "/host-boot", - ReadOnly: true, - }, - { - Name: "host-os-release", - MountPath: "/host-etc/os-release", - ReadOnly: true, - }, - { - Name: "host-sys", - MountPath: "/host-sys", - ReadOnly: true, - }, - { - Name: "host-usr-lib", - MountPath: "/host-usr/lib", - ReadOnly: true, - }, - { - Name: "host-usr-src", - MountPath: "/host-usr/src", - ReadOnly: true, - }, - }, - }, - }, - ServiceAccountName: "nfd-master-e2e", - DNSPolicy: v1.DNSClusterFirstWithHostNet, - Volumes: []v1.Volume{ - { - Name: "host-boot", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/boot", - Type: newHostPathType(v1.HostPathDirectory), - }, - }, - }, - { - Name: "host-os-release", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/os-release", - Type: newHostPathType(v1.HostPathFile), - }, - }, - }, - { - Name: "host-sys", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/sys", - Type: newHostPathType(v1.HostPathDirectory), - }, - }, - }, - { - Name: "host-usr-lib", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/usr/lib", - Type: newHostPathType(v1.HostPathDirectory), - }, - }, - }, - { - Name: "host-usr-src", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/usr/src", - Type: newHostPathType(v1.HostPathDirectory), - }, - }, - }, - }, - } - -} - -func newHostPathType(typ v1.HostPathType) *v1.HostPathType { - hostPathType := new(v1.HostPathType) - *hostPathType = v1.HostPathType(typ) - return hostPathType -} - // cleanupNode deletes all NFD-related metadata from the Node object, i.e. // labels and annotations func cleanupNode(cs clientset.Interface) { @@ -466,18 +101,17 @@ var _ = SIGDescribe("Node Feature Discovery", func() { var masterPod *v1.Pod BeforeEach(func() { - err := configureRBAC(f.ClientSet, f.Namespace.Name) + err := testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name) Expect(err).NotTo(HaveOccurred()) // Launch nfd-master By("Creating nfd master pod and nfd-master service") image := fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) - masterPod = nfdMasterPod(image, false) - masterPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), masterPod, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) + masterPod = f.PodClient().CreateSync(testutils.NFDMasterPod(image, false)) + // Create nfd-master service // Create nfd-master service - nfdSvc, err := createService(f.ClientSet, f.Namespace.Name) + nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) Expect(err).NotTo(HaveOccurred()) By("Waiting for the nfd-master pod to be running") @@ -488,7 +122,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { }) AfterEach(func() { - err := deconfigureRBAC(f.ClientSet, f.Namespace.Name) + err := testutils.DeconfigureRBAC(f.ClientSet, f.Namespace.Name) Expect(err).NotTo(HaveOccurred()) }) @@ -511,7 +145,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-worker By("Creating a nfd worker pod") image := fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) - workerPod := nfdWorkerPod(image, []string{"--oneshot", "--sources=fake"}) + workerPod := testutils.NFDWorkerPod(image, []string{"--oneshot", "--sources=fake"}) workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -547,21 +181,23 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Context("and nfd-workers as a daemonset with default sources enabled", func() { It("the node labels and annotations listed in the e2e config should be present", func() { - readConfig() - if conf == nil { + err := testutils.ReadConfig() + Expect(err).ToNot(HaveOccurred()) + + if testutils.E2EConfigFile == nil { Skip("no e2e-config was specified") } - if conf.DefaultFeatures == nil { + if testutils.E2EConfigFile.DefaultFeatures == nil { Skip("no 'defaultFeatures' specified in e2e-config") } - fConf := conf.DefaultFeatures + fConf := testutils.E2EConfigFile.DefaultFeatures // Remove pre-existing stale annotations and labels cleanupNode(f.ClientSet) By("Creating nfd-worker daemonset") - workerDS := nfdWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}) - workerDS, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) + workerDS := testutils.NFDWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}) + workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) By("Waiting for daemonset pods to be ready") @@ -572,14 +208,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { Expect(err).NotTo(HaveOccurred()) for _, node := range nodeList.Items { - var nodeConf *nodeConfig - for _, conf := range fConf.Nodes { - if conf.nameRe.MatchString(node.Name) { - e2elog.Logf("node %q matches rule %q", node.Name, conf.nameRe) - nodeConf = &conf - break - } - } + nodeConf := testutils.FindNodeConfig(node.Name) if nodeConf == nil { e2elog.Logf("node %q has no matching rule in e2e-config, skipping...", node.Name) continue @@ -717,7 +346,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { Expect(err).NotTo(HaveOccurred()) By("Creating nfd-worker daemonset with configmap mounted") - workerDS := nfdWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}) + workerDS := testutils.NFDWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}) // add configmap mount config volumeName1 := "custom-configs-extra1" @@ -801,10 +430,3 @@ var _ = SIGDescribe("Node Feature Discovery", func() { }) }) - -func pullPolicy() v1.PullPolicy { - if *pullIfNotPresent { - return v1.PullIfNotPresent - } - return v1.PullAlways -} diff --git a/test/e2e/utils/config.go b/test/e2e/utils/config.go new file mode 100644 index 0000000000..57fdec91c1 --- /dev/null +++ b/test/e2e/utils/config.go @@ -0,0 +1,104 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "flag" + "io/ioutil" + "regexp" + + e2elog "k8s.io/kubernetes/test/e2e/framework/log" + + "gopkg.in/yaml.v2" +) + +var ( + configContent = flag.String("nfd.e2e-config", "", "Configuration parameters for end-to-end tests") + E2EConfigFile *e2eConfig +) + +type e2eConfig struct { + DefaultFeatures *struct { + LabelWhitelist lookupMap + AnnotationWhitelist lookupMap + Nodes map[string]NodeConfig + } +} + +type NodeConfig struct { + ExpectedLabelValues map[string]string + ExpectedLabelKeys lookupMap + ExpectedAnnotationValues map[string]string + ExpectedAnnotationKeys lookupMap + + nameRe *regexp.Regexp +} + +type lookupMap map[string]struct{} + +func (l *lookupMap) UnmarshalJSON(data []byte) error { + *l = lookupMap{} + + var slice []string + if err := yaml.Unmarshal(data, &slice); err != nil { + return err + } + + for _, k := range slice { + (*l)[k] = struct{}{} + } + return nil +} + +func ReadConfig() error { + // Read and parse only once + if E2EConfigFile != nil || *configContent == "" { + return nil + } + + data, err := ioutil.ReadFile(*configContent) + if err != nil { + return err + } + + if err := yaml.Unmarshal(data, E2EConfigFile); err != nil { + return err + } + + // Pre-compile node name matching regexps + for name, nodeConf := range E2EConfigFile.DefaultFeatures.Nodes { + nodeConf.nameRe, err = regexp.Compile(name) + if err != nil { + return err + } + E2EConfigFile.DefaultFeatures.Nodes[name] = nodeConf + } + + return nil +} + +func FindNodeConfig(nodeName string) *NodeConfig { + var nodeConf *NodeConfig + for _, conf := range E2EConfigFile.DefaultFeatures.Nodes { + if conf.nameRe.MatchString(nodeName) { + e2elog.Logf("node %q matches rule %q", nodeName, conf.nameRe) + nodeConf = &conf + break + } + } + return nodeConf +} diff --git a/test/e2e/utils/pod.go b/test/e2e/utils/pod.go new file mode 100644 index 0000000000..cd17fba0d6 --- /dev/null +++ b/test/e2e/utils/pod.go @@ -0,0 +1,199 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "flag" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" +) + +var pullIfNotPresent = flag.Bool("nfd.pull-if-not-present", false, "Pull Images if not present - not always") + +// NFDMasterPod provide NFD master pod definition +func NFDMasterPod(image string, onMasterNode bool) *v1.Pod { + p := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "nfd-master-", + Labels: map[string]string{"name": "nfd-master-e2e"}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "node-feature-discovery", + Image: image, + ImagePullPolicy: pullPolicy(), + Command: []string{"nfd-master"}, + Env: []v1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + }, + }, + ServiceAccountName: "nfd-master-e2e", + RestartPolicy: v1.RestartPolicyNever, + }, + } + if onMasterNode { + p.Spec.NodeSelector = map[string]string{"node-role.kubernetes.io/master": ""} + p.Spec.Tolerations = []v1.Toleration{ + { + Key: "node-role.kubernetes.io/master", + Operator: v1.TolerationOpEqual, + Value: "", + Effect: v1.TaintEffectNoSchedule, + }, + } + } + return p +} + +// NFDWorkerPod provides NFD worker pod definition +func NFDWorkerPod(image string, extraArgs []string) *v1.Pod { + p := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nfd-worker-" + string(uuid.NewUUID()), + }, + Spec: *nfdWorkerPodSpec(image, extraArgs), + } + + p.Spec.RestartPolicy = v1.RestartPolicyNever + + return p +} + +// NFDWorkerDaemonSet provides the NFD daemon set worker definition +func NFDWorkerDaemonSet(image string, extraArgs []string) *appsv1.DaemonSet { + podSpec := nfdWorkerPodSpec(image, extraArgs) + return newDaemonSet("nfd-worker", podSpec) +} + +// newDaemonSet provide the new daemon set +func newDaemonSet(name string, podSpec *v1.PodSpec) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name + "-" + string(uuid.NewUUID()), + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"name": name}, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"name": name}, + }, + Spec: *podSpec, + }, + MinReadySeconds: 5, + }, + } +} + +func nfdWorkerPodSpec(image string, extraArgs []string) *v1.PodSpec { + return &v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "node-feature-discovery", + Image: image, + ImagePullPolicy: pullPolicy(), + Command: []string{"nfd-worker"}, + Args: append([]string{"--server=nfd-master-e2e:8080"}, extraArgs...), + Env: []v1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "host-boot", + MountPath: "/host-boot", + ReadOnly: true, + }, + { + Name: "host-os-release", + MountPath: "/host-etc/os-release", + ReadOnly: true, + }, + { + Name: "host-sys", + MountPath: "/host-sys", + ReadOnly: true, + }, + }, + }, + }, + ServiceAccountName: "nfd-master-e2e", + DNSPolicy: v1.DNSClusterFirstWithHostNet, + Volumes: []v1.Volume{ + { + Name: "host-boot", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/boot", + Type: newHostPathType(v1.HostPathDirectory), + }, + }, + }, + { + Name: "host-os-release", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/os-release", + Type: newHostPathType(v1.HostPathFile), + }, + }, + }, + { + Name: "host-sys", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/sys", + Type: newHostPathType(v1.HostPathDirectory), + }, + }, + }, + }, + } + +} + +func newHostPathType(typ v1.HostPathType) *v1.HostPathType { + hostPathType := new(v1.HostPathType) + *hostPathType = typ + return hostPathType +} + +func pullPolicy() v1.PullPolicy { + if *pullIfNotPresent { + return v1.PullIfNotPresent + } + return v1.PullAlways +} diff --git a/test/e2e/utils/rbac.go b/test/e2e/utils/rbac.go new file mode 100644 index 0000000000..157e5600ff --- /dev/null +++ b/test/e2e/utils/rbac.go @@ -0,0 +1,138 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "flag" + + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" +) + +var ( + openShift = flag.Bool("nfd.openshift", false, "Enable OpenShift specific bits") +) + +// ConfigureRBAC creates required RBAC configuration +func ConfigureRBAC(cs clientset.Interface, ns string) error { + _, err := createServiceAccount(cs, ns) + if err != nil { + return err + } + + _, err = createClusterRole(cs) + if err != nil { + return err + } + + _, err = createClusterRoleBinding(cs, ns) + if err != nil { + return err + } + + return nil +} + +// DeconfigureRBAC removes RBAC configuration +func DeconfigureRBAC(cs clientset.Interface, ns string) error { + err := cs.RbacV1().ClusterRoleBindings().Delete(context.TODO(), "nfd-master-e2e", metav1.DeleteOptions{}) + if err != nil { + return err + } + err = cs.RbacV1().ClusterRoles().Delete(context.TODO(), "nfd-master-e2e", metav1.DeleteOptions{}) + if err != nil { + return err + } + err = cs.CoreV1().ServiceAccounts(ns).Delete(context.TODO(), "nfd-master-e2e", metav1.DeleteOptions{}) + if err != nil { + return err + } + return nil +} + +// Configure service account required by NFD +func createServiceAccount(cs clientset.Interface, ns string) (*v1.ServiceAccount, error) { + sa := &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nfd-master-e2e", + Namespace: ns, + }, + } + return cs.CoreV1().ServiceAccounts(ns).Create(context.TODO(), sa, metav1.CreateOptions{}) +} + +// Configure cluster role required by NFD +func createClusterRole(cs clientset.Interface) (*rbacv1.ClusterRole, error) { + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nfd-master-e2e", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "patch", "update"}, + }, + { + APIGroups: []string{"topology.node.k8s.io"}, + Resources: []string{"noderesourcetopologies"}, + Verbs: []string{ + "create", + "get", + "update", + }, + }, + }, + } + if *openShift { + cr.Rules = append(cr.Rules, + rbacv1.PolicyRule{ + // needed on OpenShift clusters + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{"hostaccess"}, + Verbs: []string{"use"}, + }) + } + return cs.RbacV1().ClusterRoles().Update(context.TODO(), cr, metav1.UpdateOptions{}) +} + +// Configure cluster role binding required by NFD +func createClusterRoleBinding(cs clientset.Interface, ns string) (*rbacv1.ClusterRoleBinding, error) { + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nfd-master-e2e", + }, + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: "nfd-master-e2e", + Namespace: ns, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: "nfd-master-e2e", + }, + } + + return cs.RbacV1().ClusterRoleBindings().Update(context.TODO(), crb, metav1.UpdateOptions{}) +} diff --git a/test/e2e/utils/service.go b/test/e2e/utils/service.go new file mode 100644 index 0000000000..1ea3b08d45 --- /dev/null +++ b/test/e2e/utils/service.go @@ -0,0 +1,45 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" +) + +// CreateService creates nfd-master Service +func CreateService(cs clientset.Interface, ns string) (*v1.Service, error) { + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nfd-master-e2e", + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{"name": "nfd-master-e2e"}, + Ports: []v1.ServicePort{ + { + Protocol: v1.ProtocolTCP, + Port: 8080, + }, + }, + Type: v1.ServiceTypeClusterIP, + }, + } + return cs.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{}) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2b40b38cf2..789cb53552 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -923,6 +923,7 @@ gopkg.in/tomb.v1 # gopkg.in/warnings.v0 v0.1.1 gopkg.in/warnings.v0 # gopkg.in/yaml.v2 v2.4.0 +## explicit gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c gopkg.in/yaml.v3