Add DRA Integration E2E test

Signed-off-by: Vasilis Remmas <[email protected]>

.github/workflows/kind-e2e.yml

@@ -85,6 +85,10 @@ jobs:
         working-directory: ./e2e
         run: ./test-default-route1.sh
 
+      - name: Test DRA integration
+        working-directory: ./e2e
+        run: ./test-dra-integration.sh
+
       - name: Export kind logs
         if: always()
         run: |

.gitignore

@@ -2,6 +2,7 @@
 bin/
 e2e/bin/
 e2e/yamls/
+e2e/repos/
 
 # GOPATH created by the build script
 gopath/

e2e/get_tools.sh

@@ -13,3 +13,4 @@ curl -Lo ./bin/koko https://github.com/redhat-nfvpe/koko/releases/download/v0.83
 chmod +x ./bin/koko
 curl -Lo ./bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
 chmod +x ./bin/jq
+wget -qO- https://get.helm.sh/helm-v3.14.3-linux-amd64.tar.gz | tar xvzf - --strip-components=1 -C ./bin linux-amd64/helm

e2e/setup_cluster.sh

@@ -34,7 +34,21 @@ nodes:
       nodeRegistration:
         kubeletExtraArgs:
           pod-manifest-path: "/etc/kubernetes/manifests/"
+          feature-gates: "DynamicResourceAllocation=true,KubeletPodResourcesDynamicResources=true"
   - role: worker
+# Required by DRA Integration
+##
+featureGates:
+  DynamicResourceAllocation: true
+runtimeConfig:
+  "api/alpha": "true"
+containerdConfigPatches:
+# Enable CDI as described in
+# https://github.com/container-orchestrated-devices/container-device-interface#containerd-configuration
+- |-
+  [plugins."io.containerd.grpc.v1.cri"]
+      enable_cdi = true
+##
 EOF
 
 # load multus image from container host to kind node

e2e/templates/dra-integration.yml.j2

@@ -0,0 +1,49 @@
+---
+apiVersion: resource.k8s.io/v1alpha2
+kind: ResourceClaimTemplate
+metadata:
+  name: gpu.example.com
+spec:
+  spec:
+    resourceClassName: gpu.example.com
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: dra-net 
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: gpu.example.com
+spec:
+  config: '{
+        "cniVersion": "{{ CNI_VERSION }}",
+        "plugins": [{
+            "name": "mynet",
+            "type": "dummy",
+            "ipam": {
+                "type": "host-local",
+                "subnet": "10.1.2.0/24"
+            }
+        }]
+    }'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dra-integration
+  labels:
+    app: dra-integration
+  annotations:
+    k8s.v1.cni.cncf.io/networks: default/dra-net
+spec:
+  containers:
+  - name: ctr0
+    image: ubuntu:22.04
+    command: ["bash", "-c"]
+    args: ["export; sleep 9999"]
+    resources:
+      claims:
+      - name: gpu
+  resourceClaims:
+  - name: gpu
+    source:
+      resourceClaimTemplateName: gpu.example.com

e2e/templates/multus-daemonset-thick.yml.j2

@@ -158,6 +158,9 @@ spec:
         - name: multus-daemon-config
           mountPath: /etc/cni/net.d/multus.d
           readOnly: true
+        - name: kubelet-pod-resources
+          mountPath: /var/lib/kubelet/pod-resources
+          readOnly: true
         env:
         - name: MULTUS_NODE_NAME
           valueFrom:
@@ -187,6 +190,9 @@ spec:
         - name: cnibin
           hostPath:
             path: /opt/cni/bin
+        - name: kubelet-pod-resources
+          hostPath:
+            path: /var/lib/kubelet/pod-resources
         - name: multus-daemon-config
           configMap:
             name: multus-daemon-config

e2e/test-dra-integration.sh

@@ -0,0 +1,59 @@
+#!/bin/sh
+set -o errexit
+
+export PATH=${PATH}:./bin
+
+# This test is using an example implementation of a DRA driver. This driver is mocking GPU resources. At our test we
+# don't care about what these resources are. We want to ensure that such resource is correctly passed in the Pod using
+# Multus configurations. A couple of notes:
+# - We explitictly don't pin the revision of the dra-example-driver to a specific commit to ensure that the integration
+#   continues to work even when the dra-example-driver is updated (which may also indicate API changes on the DRA).
+# - The chart and latest is image is not published somewhere, therefore we have to build locally. This leads to slower
+#   e2e suite runs.
+echo "installing dra-example-driver"
+repo_path="repos/dra-example-driver"
+
+rm -rf $repo_path || true
+git clone https://github.com/kubernetes-sigs/dra-example-driver.git ${repo_path}
+${repo_path}/demo/build-driver.sh
+KIND_CLUSTER_NAME=kind ${repo_path}/demo/scripts/load-driver-image-into-kind.sh
+chart_path=${repo_path}/deployments/helm/dra-example-driver/
+overriden_values_path=${chart_path}/overriden_values.yaml
+
+# With the thick plugin, in kind, the primary network on the control plane is not always working as expected. The pods
+# sometimes are not able to communicate with the control plane and the error looks like this:
+# failed to list *v1alpha2.PodSchedulingContext: Get "https://10.96.0.1:443/apis/resource.k8s.io/v1alpha2/podschedulingcontexts?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
+# We override the values here to schedule the controller on the worker nodes where the network is working as expected.
+cat <<EOF >> ${overriden_values_path}
+controller:
+  nodeSelector: null
+  tolerations: null
+EOF
+
+helm install \
+    -n dra-example-driver \
+    --create-namespace \
+    -f ${overriden_values_path} \
+    dra-example-driver \
+    ${chart_path}
+
+echo "installing testing pods"
+kubectl create -f yamls/dra-integration.yml
+kubectl wait --for=condition=ready -l app=dra-integration --timeout=300s pod
+
+echo "check dra-integration pod for DRA injected environment variable"
+
+# We can validate that the resource is correctly injected by checking an environment variable this dra driver is injecting
+# in the Pod.
+# https://github.com/kubernetes-sigs/dra-example-driver/blob/be2b8b1db47b8c757440e955ce5ced88c23bfe86/cmd/dra-example-kubeletplugin/cdi.go#L71C20-L71C44
+env_variable=$(kubectl exec dra-integration -- bash -c "echo \$DRA_RESOURCE_DRIVER_NAME | grep gpu.resource.example.com")
+if [ $? -eq 0 ];then
+	echo "dra-integration pod has DRA injected environment variable"
+else
+	echo "dra-integration pod doesn't have DRA injected environment variable"
+	exit 1
+fi
+
+echo "cleanup resources"
+kubectl delete -f yamls/dra-integration.yml
+helm uninstall -n dra-example-driver dra-example-driver