Skip to content

Commit

Permalink
test/e2e: add basic e2e-tests for NodeFeature API
Browse files Browse the repository at this point in the history 
Add an initial test set for the NodeFeature API. This is done simply by
running a second pass of the tests but with -enable-nodefeature-api
(i.e. NodeFeature API enabled and gRPC disabled). This should give basic
confidence that the API actually works and form a basis for further
imporovements on testing the new CRD API.
  • Loading branch information
@marquiz
marquiz committed Dec 19, 2022
1 parent 958db56 commit b67d6d7
Show file tree
Hide file tree
Showing 3 changed files with 116 additions and 32 deletions.
52 changes: 39 additions & 13 deletions test/e2e/node_feature_discovery.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,14 +154,31 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
f := framework.NewDefaultFramework("node-feature-discovery")

nfdTestSuite := func(useNodeFeatureApi bool) {
createPodSpecOpts := func(opts ...testpod.SpecOption) []testpod.SpecOption {
if useNodeFeatureApi {
return append(opts, testpod.SpecWithContainerExtraArgs("-enable-nodefeature-api"))
}
return opts
}

Context("when deploying a single nfd-master pod", Ordered, func() {
var (
masterPod *corev1.Pod
crds []*apiextensionsv1.CustomResourceDefinition
extClient *extclient.Clientset
nfdClient *nfdclient.Clientset
)

checkNodeFeatureObject := func(name string) {
_, err := nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Get(context.TODO(), name, metav1.GetOptions{})
if useNodeFeatureApi {
By(fmt.Sprintf("Check that NodeFeature object for the node %q was created", name))
Expect(err).NotTo(HaveOccurred())
} else {
By(fmt.Sprintf("Check that NodeFeature object for the node %q hasn't been created", name))
Expect(err).To(HaveOccurred())
}
}

BeforeAll(func() {
// Create clients for apiextensions and our CRD api
extClient = extclient.NewForConfigOrDie(f.ClientConfig())
Expand Down Expand Up @@ -198,13 +215,12 @@ var _ = SIGDescribe("Node Feature Discovery", func() {

// Launch nfd-master
By("Creating nfd master pod and nfd-master service")

imageOpt := []testpod.SpecOption{
podSpecOpts := createPodSpecOpts(
testpod.SpecWithContainerImage(dockerImage),
testpod.SpecWithTolerations(testTolerations),
testpod.SpecWithContainerExtraArgs("-enable-taints"),
}
masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt...))
)
masterPod := e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(podSpecOpts...))

// Create nfd-master service
nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name)
Expand Down Expand Up @@ -247,12 +263,12 @@ var _ = SIGDescribe("Node Feature Discovery", func() {

// Launch nfd-worker
By("Creating a nfd worker pod")
podSpecOpts := []testpod.SpecOption{
podSpecOpts := createPodSpecOpts(
testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever),
testpod.SpecWithContainerImage(dockerImage),
testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"),
testpod.SpecWithTolerations(testTolerations),
}
)
workerPod := testpod.NFDWorker(podSpecOpts...)
workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
Expand All @@ -276,6 +292,8 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
}
}

checkNodeFeatureObject(node.Name)

By("Deleting the node-feature-discovery worker pod")
err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), workerPod.ObjectMeta.Name, metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
Expand All @@ -299,10 +317,10 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
fConf := cfg.DefaultFeatures

By("Creating nfd-worker daemonset")
podSpecOpts := []testpod.SpecOption{
podSpecOpts := createPodSpecOpts(
testpod.SpecWithContainerImage(dockerImage),
testpod.SpecWithTolerations(testTolerations),
}
)
workerDS := testds.NFDWorker(podSpecOpts...)
workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
Expand Down Expand Up @@ -364,6 +382,9 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
}
}

// Check existence of NodeFeature object
checkNodeFeatureObject(node.Name)

}

By("Deleting nfd-worker daemonset")
Expand Down Expand Up @@ -426,12 +447,12 @@ var _ = SIGDescribe("Node Feature Discovery", func() {
Expect(err).NotTo(HaveOccurred())

By("Creating nfd-worker daemonset with configmap mounted")
podSpecOpts := []testpod.SpecOption{
podSpecOpts := createPodSpecOpts(
testpod.SpecWithContainerImage(dockerImage),
testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")),
testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")),
testpod.SpecWithTolerations(testTolerations),
}
)
workerDS := testds.NFDWorker(podSpecOpts...)

workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
Expand Down Expand Up @@ -489,11 +510,11 @@ core:
Expect(err).NotTo(HaveOccurred())

By("Creating nfd-worker daemonset")
podSpecOpts := []testpod.SpecOption{
podSpecOpts := createPodSpecOpts(
testpod.SpecWithContainerImage(dockerImage),
testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"),
testpod.SpecWithTolerations(testTolerations),
}
)
workerDS := testds.NFDWorker(podSpecOpts...)
workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
Expand Down Expand Up @@ -580,6 +601,11 @@ core:
nfdTestSuite(false)

})

Context("when running NFD with NodeFeature CRD API enabled", func() {
nfdTestSuite(true)
})

})

// waitForNfdNodeAnnotations waits for node to be annotated as expected.
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/utils/pod/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ func nfdWorkerSpec(opts ...SpecOption) *corev1.PodSpec {
},
},
},
ServiceAccountName: "nfd-master-e2e",
ServiceAccountName: "nfd-worker-e2e",
DNSPolicy: corev1.DNSClusterFirstWithHostNet,
Volumes: []corev1.Volume{
{
Expand Down
94 changes: 76 additions & 18 deletions test/e2e/utils/rbac.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,17 @@ var (

// ConfigureRBAC creates required RBAC configuration
func ConfigureRBAC(cs clientset.Interface, ns string) error {
_, err := createServiceAccountMaster(cs, ns)
_, err := createServiceAccount(cs, "nfd-master-e2e", ns)
if err != nil {
return err
}

_, err = createServiceAccountTopologyUpdater(cs, ns)
_, err = createServiceAccount(cs, "nfd-worker-e2e", ns)
if err != nil {
return err
}

_, err = createServiceAccount(cs, "nfd-topology-updater-e2e", ns)
if err != nil {
return err
}
Expand All @@ -47,6 +52,11 @@ func ConfigureRBAC(cs clientset.Interface, ns string) error {
return err
}

_, err = createRoleWorker(cs, ns)
if err != nil {
return err
}

_, err = createClusterRoleTopologyUpdater(cs)
if err != nil {
return err
Expand All @@ -57,6 +67,11 @@ func ConfigureRBAC(cs clientset.Interface, ns string) error {
return err
}

_, err = createRoleBindingWorker(cs, ns)
if err != nil {
return err
}

_, err = createClusterRoleBindingTopologyUpdater(cs, ns)
if err != nil {
return err
Expand All @@ -75,6 +90,10 @@ func DeconfigureRBAC(cs clientset.Interface, ns string) error {
if err != nil {
return err
}
err = cs.RbacV1().RoleBindings(ns).Delete(context.TODO(), "nfd-worker-e2e", metav1.DeleteOptions{})
if err != nil {
return err
}
err = cs.RbacV1().ClusterRoles().Delete(context.TODO(), "nfd-topology-updater-e2e", metav1.DeleteOptions{})
if err != nil {
return err
Expand All @@ -83,6 +102,10 @@ func DeconfigureRBAC(cs clientset.Interface, ns string) error {
if err != nil {
return err
}
err = cs.RbacV1().Roles(ns).Delete(context.TODO(), "nfd-worker-e2e", metav1.DeleteOptions{})
if err != nil {
return err
}
err = cs.CoreV1().ServiceAccounts(ns).Delete(context.TODO(), "nfd-topology-updater-e2e", metav1.DeleteOptions{})
if err != nil {
return err
Expand All @@ -91,25 +114,18 @@ func DeconfigureRBAC(cs clientset.Interface, ns string) error {
if err != nil {
return err
}
return nil
}

// Configure service account required by NFD Master
func createServiceAccountMaster(cs clientset.Interface, ns string) (*corev1.ServiceAccount, error) {
sa := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "nfd-master-e2e",
Namespace: ns,
},
err = cs.CoreV1().ServiceAccounts(ns).Delete(context.TODO(), "nfd-worker-e2e", metav1.DeleteOptions{})
if err != nil {
return err
}
return cs.CoreV1().ServiceAccounts(ns).Create(context.TODO(), sa, metav1.CreateOptions{})
return nil
}

// Configure service account required by NFD MTopology Updater
func createServiceAccountTopologyUpdater(cs clientset.Interface, ns string) (*corev1.ServiceAccount, error) {
// Configure service account
func createServiceAccount(cs clientset.Interface, name, ns string) (*corev1.ServiceAccount, error) {
sa := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "nfd-topology-updater-e2e",
Name: name,
Namespace: ns,
},
}
Expand All @@ -126,11 +142,11 @@ func createClusterRoleMaster(cs clientset.Interface) (*rbacv1.ClusterRole, error
{
APIGroups: []string{""},
Resources: []string{"nodes"},
Verbs: []string{"get", "patch", "update"},
Verbs: []string{"get", "list", "patch", "update"},
},
{
APIGroups: []string{"nfd.k8s-sigs.io"},
Resources: []string{"nodefeaturerules"},
Resources: []string{"nodefeatures", "nodefeaturerules"},
Verbs: []string{"get", "list", "watch"},
},
},
Expand All @@ -148,6 +164,24 @@ func createClusterRoleMaster(cs clientset.Interface) (*rbacv1.ClusterRole, error
return cs.RbacV1().ClusterRoles().Update(context.TODO(), cr, metav1.UpdateOptions{})
}

// Configure role required by NFD Worker
func createRoleWorker(cs clientset.Interface, ns string) (*rbacv1.Role, error) {
cr := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: "nfd-worker-e2e",
Namespace: ns,
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{"nfd.k8s-sigs.io"},
Resources: []string{"nodefeatures"},
Verbs: []string{"create", "get", "update"},
},
},
}
return cs.RbacV1().Roles(ns).Update(context.TODO(), cr, metav1.UpdateOptions{})
}

// Configure cluster role required by NFD Topology Updater
func createClusterRoleTopologyUpdater(cs clientset.Interface) (*rbacv1.ClusterRole, error) {
cr := &rbacv1.ClusterRole{
Expand Down Expand Up @@ -210,6 +244,30 @@ func createClusterRoleBindingMaster(cs clientset.Interface, ns string) (*rbacv1.
return cs.RbacV1().ClusterRoleBindings().Update(context.TODO(), crb, metav1.UpdateOptions{})
}

// Configure role binding required by NFD Master
func createRoleBindingWorker(cs clientset.Interface, ns string) (*rbacv1.RoleBinding, error) {
crb := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "nfd-worker-e2e",
Namespace: ns,
},
Subjects: []rbacv1.Subject{
{
Kind: rbacv1.ServiceAccountKind,
Name: "nfd-worker-e2e",
Namespace: ns,
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: rbacv1.GroupName,
Kind: "Role",
Name: "nfd-worker-e2e",
},
}

return cs.RbacV1().RoleBindings(ns).Update(context.TODO(), crb, metav1.UpdateOptions{})
}

// Configure cluster role binding required by NFD Topology Updater
func createClusterRoleBindingTopologyUpdater(cs clientset.Interface, ns string) (*rbacv1.ClusterRoleBinding, error) {
crb := &rbacv1.ClusterRoleBinding{
Expand Down

0 comments on commit b67d6d7

Please sign in to comment.