WIP: Use intel-gpu-plugin with intel-gpu-fakedev generated devices

TODO: Adding new kustomization (base) directory for this is alternative for kustomizing existing gpu_plugin/ base. It's needed until there's a way for overlay to specify which GPU plugin initcontainer gets run first. Signed-off-by: Eero Tamminen <[email protected]>
eero-t · Aug 25, 2022 · efc4e48 · efc4e48
1 parent c190d6d
commit efc4e48
Show file tree

Hide file tree

Showing 4 changed files with 123 additions and 0 deletions.
diff --git a/deployments/gpu_fakedev/base/fakedev-config.json b/deployments/gpu_fakedev/base/fakedev-config.json
@@ -0,0 +1,8 @@
+{
+	"Info": "8x 4 GiB DG1 [Iris Xe MAX Graphics] GPUs",
+	"DevCount": 8,
+	"DevMemSize": 4294967296,
+	"Capabilities": {
+		"platform": "fake_DG1",
+	}
+}
diff --git a/deployments/gpu_fakedev/base/intel-gpu-plugin.yaml b/deployments/gpu_fakedev/base/intel-gpu-plugin.yaml
@@ -0,0 +1,107 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-gpu-plugin-fake
+  labels:
+    app: intel-gpu-plugin-fake
+spec:
+  selector:
+    matchLabels:
+      app: intel-gpu-plugin-fake
+  template:
+    metadata:
+      labels:
+        app: intel-gpu-plugin-fake
+    spec:
+      initContainers:
+      - name: fakedev-generator
+        image: intel/intel-gpu-fakedev:devel
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          readOnlyRootFilesystem: false
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: [ "ALL" ]
+            add:  [ "MKNOD" ]
+        # container runtime prevents writing to /sys & /dev,
+        # so volumes need to be mounted elsewhere
+        volumeMounts:
+        - name: devfs
+          mountPath: /tmp/fakedev/dev
+          readOnly: false
+        - name: sysfs
+          mountPath: /tmp/fakedev/sys
+          readOnly: false
+        - name: fake-conf
+          mountPath: /config
+          readOnly: true
+        # files are generated under CWD
+        workingDir: /tmp/fakedev
+        # generate fake sysfs / devfs files for GPU plugin based on config
+        command: ["/generator", "-json", "/config/fakedev.json", "-verbose"]
+      - name: intel-gpu-initcontainer
+        # convert generated sysfs content to NFD feature labels file
+        image: intel/intel-gpu-initcontainer:devel
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: [ "ALL" ]
+        # expects sysfs here
+        volumeMounts:
+        - name: sysfs
+          mountPath: /host-sys
+          readOnly: true
+        - name: nfd-features
+          mountPath: /nfd
+          readOnly: false
+        workingDir: /usr/local/bin/gpu-sw
+        # needed until GPU plugin drops NFD hook usage due to:
+        # https://github.com/kubernetes-sigs/node-feature-discovery/issues/856
+        command: ["sh", "-c", "./intel-gpu-nfdhook | tee /nfd/fake-gpu"]
+      containers:
+      - name: intel-gpu-plugin
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+        image: intel/intel-gpu-plugin:devel
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: [ "ALL" ]
+        args: [ "-prefix=/tmp/fakedev" ]
+        # devfs host & container paths must match for everything to work
+        volumeMounts:
+        - name: devfs
+          mountPath: /tmp/fakedev/dev
+          readOnly: true
+        - name: sysfs
+          mountPath: /tmp/fakedev/sys
+          readOnly: true
+        - name: kubeletsockets
+          mountPath: /var/lib/kubelet/device-plugins
+      volumes:
+      - name: devfs
+        hostPath:
+          path: /tmp/fakedev/dev
+          type: DirectoryOrCreate
+      - name: sysfs
+        hostPath:
+        emptyDir: {}
+      - name: fake-conf
+        configMap:
+          name: fakedev-config
+      - name: kubeletsockets
+        hostPath:
+          path: /var/lib/kubelet/device-plugins
+      - name: nfd-features
+        hostPath:
+          path: /etc/kubernetes/node-feature-discovery/features.d/
+          type: DirectoryOrCreate
+      nodeSelector:
+        kubernetes.io/arch: amd64
diff --git a/deployments/gpu_fakedev/base/kustomization.yaml b/deployments/gpu_fakedev/base/kustomization.yaml
@@ -0,0 +1,6 @@
+configMapGenerator:
+- name: fakedev-config
+  files:
+  - fakedev-config.json
+resources:
+  - intel-gpu-plugin.yaml
diff --git a/deployments/gpu_fakedev/kustomization.yaml b/deployments/gpu_fakedev/kustomization.yaml
@@ -0,0 +1,2 @@
+bases:
+  - base