diff --git a/build/docker/intel-qat-initcontainer.Dockerfile b/build/docker/intel-qat-initcontainer.Dockerfile index e5be59ec3..08da90a02 100644 --- a/build/docker/intel-qat-initcontainer.Dockerfile +++ b/build/docker/intel-qat-initcontainer.Dockerfile @@ -62,4 +62,5 @@ LABEL summary='Intel® QAT initcontainer for Kubernetes' LABEL description='Intel QAT initcontainer initializes devices' COPY --from=builder /install_root / COPY demo/qat-init.sh /usr/local/bin/ +WORKDIR /qat-init ENTRYPOINT [ "/bin/bash", "/usr/local/bin/qat-init.sh"] diff --git a/build/docker/templates/intel-qat-initcontainer.Dockerfile.in b/build/docker/templates/intel-qat-initcontainer.Dockerfile.in index 77bd77fc9..49ae9ad99 100644 --- a/build/docker/templates/intel-qat-initcontainer.Dockerfile.in +++ b/build/docker/templates/intel-qat-initcontainer.Dockerfile.in @@ -21,4 +21,6 @@ COPY --from=builder /install_root / COPY demo/qat-init.sh /usr/local/bin/ +WORKDIR /qat-init + ENTRYPOINT [ "/bin/bash", "/usr/local/bin/qat-init.sh"] diff --git a/build/docker/toybox-config b/build/docker/toybox-config index e6e581331..a974b874f 100644 --- a/build/docker/toybox-config +++ b/build/docker/toybox-config @@ -1,7 +1,7 @@ # # Automatically generated make config: don't edit # ToyBox version: KCONFIG_VERSION -# Tue Oct 11 13:47:43 2022 +# Thu Nov 17 14:23:21 2022 # # CONFIG_TOYBOX_ON_ANDROID is not set CONFIG_TOYBOX_FORK=y @@ -23,7 +23,7 @@ CONFIG_CP=y # CONFIG_MV is not set # CONFIG_INSTALL is not set # CONFIG_CPIO is not set -# CONFIG_CUT is not set +CONFIG_CUT=y # CONFIG_DATE is not set # CONFIG_DF is not set # CONFIG_DIRNAME is not set diff --git a/cmd/qat_plugin/README.md b/cmd/qat_plugin/README.md index 9c546f784..c83203e41 100644 --- a/cmd/qat_plugin/README.md +++ b/cmd/qat_plugin/README.md @@ -119,6 +119,34 @@ $ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/ > socket creation and kubelet registration. Furthermore, the deployments `securityContext` must > be configured with appropriate `runAsUser/runAsGroup`. +#### Automatic Provisioning + +There's a sample [qat initcontainer](https://github.com/intel/intel-device-plugins-for-kubernetes/blob/main/build/docker/intel-qat-initcontainer.Dockerfile). Regardless of device types, the script running inside the initcontainer enables QAT SR-IOV VFs. + +To deploy, run as follows: + +```bash +$ kubectl apply -k deployments/qat_plugin/overlays/qat_initcontainer/ +``` + +In addition to the default configuration, you can add device-specific configurations via ConfigMap. + +| Device | Possible Configuration | How To Customize | Options | Notes | +|:-------|:-----------------------|:-----------------|:--------|:------| +| 4xxx, 401xx | [cfg_services](https://github.com/torvalds/linux/blob/42e66b1cc3a070671001f8a1e933a80818a192bf/Documentation/ABI/testing/sysfs-driver-qat) reports the configured services (crypto services or compression services) of the QAT device. | `ServicesEnabled=` | compress:`dc`, crypto:`sym;asym` | Linux 6.0+ kernel is required. | + +To create a provisioning config after customizing, run as follows: + +```bash +$ kubectl create configmap --namespace=inteldeviceplugins-system qat-config --from-file=deployments/qat_plugin/overlays/qat_initcontainer/qat.conf +``` +> **Note**: When deploying the overlay qat_initcontainer, such a manual creation is not necessary since ConfigMap is generated automatically. Just set the values in the config file and deploy the overlay. + +When using the operator for deploying the plugin with provisioning config, use `provisioningConfig` field for the name of the ConfigMap, then the config is passed to initcontainer through the volume mount. + +There's also a possibility for a node specific congfiguration through passing a nodename via `NODE_NAME` into initcontainer's environment and passing a node specific profile (`qat-$NODE_NAME.conf`) via ConfigMap volume mount. + + #### Verify Plugin Registration Verification of the plugin deployment and detection of QAT hardware can be confirmed by diff --git a/demo/qat-init.sh b/demo/qat-init.sh index 22af081df..2cc0c1774 100755 --- a/demo/qat-init.sh +++ b/demo/qat-init.sh @@ -1,8 +1,55 @@ -#!/bin/sh -eu +#!/bin/sh +# This script is based on qatlib's qat_init.sh +NODE_NAME="${NODE_NAME:-}" ENABLED_QAT_PF_PCIIDS=${ENABLED_QAT_PF_PCIIDS:-37c8 4940 4942} DEVS=$(for pf in $ENABLED_QAT_PF_PCIIDS; do lspci -n | grep -e "$pf" | grep -o -e "^\\S*"; done) +SERVICES_LIST="sym;asym dc" +QAT_4XXX_DEVICE_PCI_ID="0x4940" +QAT_401XX_DEVICE_PCI_ID="0x4942" +SERVICES_ENABLED="NONE" +SERVICES_ENABLED_FOUND="FALSE" -for dev in $DEVS; do +check_config() { + [ -f "conf/qat.conf" ] && SERVICES_ENABLED=$(cut -d= -f 2 conf/qat.conf | grep '\S') + [ -f "conf/qat-$NODE_NAME.conf" ] && SERVICES_ENABLED=$(cut -d= -f 2 conf/qat-$NODE_NAME.conf | grep '\S') + + if [ "$SERVICES_ENABLED" != "NONE" ]; then + SERVICES_ENABLED_FOUND="FALSE" + for SERVICE in $SERVICES_LIST + do + if [ "$SERVICE" = "$SERVICES_ENABLED" ]; then + SERVICES_ENABLED_FOUND="TRUE" + break + fi + done + fi +} + +sysfs_config() { + if [ "$SERVICES_ENABLED_FOUND" = "TRUE" ]; then + for dev in $DEVS; do + DEVPATH="/sys/bus/pci/devices/0000:$dev" + PCI_DEV=$(cat "$DEVPATH"/device 2> /dev/null) + if [ "$PCI_DEV" != "$QAT_4XXX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_401XX_DEVICE_PCI_ID" ]; then + continue + fi + + CURRENT_SERVICES=$(cat "$DEVPATH"/qat/cfg_services) + if [ "$CURRENT_SERVICES" != "$SERVICES_ENABLED" ]; then + CURRENT_STATE=$(cat "$DEVPATH"/qat/state) + if [ "$CURRENT_STATE" = "up" ]; then + echo down > "$DEVPATH"/qat/state + fi + echo "$SERVICES_ENABLED" > "$DEVPATH"/qat/cfg_services + CURRENT_SERVICES=$(cat "$DEVPATH"/qat/cfg_services) + fi + echo "Device $dev configured with services: $CURRENT_SERVICES" + done + fi +} + +enable_sriov() { + for dev in $DEVS; do DEVPATH="/sys/bus/pci/devices/0000:$dev" NUMVFS="$DEVPATH/sriov_numvfs" if ! test -w "$NUMVFS"; then @@ -14,4 +61,9 @@ for dev in $DEVS; do else tee "$NUMVFS" < "$DEVPATH/sriov_totalvfs" fi -done + done +} + +check_config +sysfs_config +enable_sriov diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml index ded8755a2..7143d3131 100644 --- a/deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml +++ b/deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml @@ -102,6 +102,10 @@ spec: - balanced - packed type: string + provisioningConfig: + description: ProvisioningConfig is a ConfigMap used to pass the configuration + of QAT devices into qat initcontainer. + type: string type: object status: description: 'QatDevicePluginStatus defines the observed state of QatDevicePlugin. diff --git a/deployments/qat_plugin/base/intel-qat-plugin.yaml b/deployments/qat_plugin/base/intel-qat-plugin.yaml index dfe9414e6..6d5678c22 100644 --- a/deployments/qat_plugin/base/intel-qat-plugin.yaml +++ b/deployments/qat_plugin/base/intel-qat-plugin.yaml @@ -16,6 +16,11 @@ spec: automountServiceAccountToken: false containers: - name: intel-qat-plugin + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName image: intel/intel-qat-plugin:devel securityContext: seLinuxOptions: diff --git a/deployments/qat_plugin/overlays/e2e/kustomization.yaml b/deployments/qat_plugin/overlays/e2e/kustomization.yaml index 58ade5759..471985c86 100644 --- a/deployments/qat_plugin/overlays/e2e/kustomization.yaml +++ b/deployments/qat_plugin/overlays/e2e/kustomization.yaml @@ -3,7 +3,7 @@ commonAnnotations: container.apparmor.security.beta.kubernetes.io/intel-qat-plugin: unconfined resources: -- ../sriov_numvfs +- ../qat_initcontainer patches: - path: add-args.yaml diff --git a/deployments/qat_plugin/overlays/qat_initcontainer/kustomization.yaml b/deployments/qat_plugin/overlays/qat_initcontainer/kustomization.yaml new file mode 100644 index 000000000..b31cf223f --- /dev/null +++ b/deployments/qat_plugin/overlays/qat_initcontainer/kustomization.yaml @@ -0,0 +1,8 @@ +bases: +- ../../base +patchesStrategicMerge: +- qat_initcontainer.yaml +configMapGenerator: +- name: qat-config + files: + - qat.conf diff --git a/deployments/qat_plugin/overlays/qat_initcontainer/qat.conf b/deployments/qat_plugin/overlays/qat_initcontainer/qat.conf new file mode 100644 index 000000000..040eef655 --- /dev/null +++ b/deployments/qat_plugin/overlays/qat_initcontainer/qat.conf @@ -0,0 +1 @@ +ServicesEnabled=sym;asym diff --git a/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml b/deployments/qat_plugin/overlays/qat_initcontainer/qat_initcontainer.yaml similarity index 55% rename from deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml rename to deployments/qat_plugin/overlays/qat_initcontainer/qat_initcontainer.yaml index 9472646bc..3dcdd9b2a 100644 --- a/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml +++ b/deployments/qat_plugin/overlays/qat_initcontainer/qat_initcontainer.yaml @@ -6,7 +6,12 @@ spec: template: spec: initContainers: - - name: sriov-numvfs + - name: intel-qat-initcontainer + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName image: intel/intel-qat-initcontainer:devel securityContext: readOnlyRootFilesystem: true @@ -14,7 +19,13 @@ spec: volumeMounts: - name: sysfs mountPath: /sys + - name: qat-config + mountPath: /qat-init/conf volumes: - name: sysfs hostPath: path: /sys + - name: qat-config + configMap: + name: qat-config + defaultMode: 0440 diff --git a/deployments/qat_plugin/overlays/sriov_numvfs/kustomization.yaml b/deployments/qat_plugin/overlays/sriov_numvfs/kustomization.yaml deleted file mode 100644 index 1fb07dcae..000000000 --- a/deployments/qat_plugin/overlays/sriov_numvfs/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -bases: -- ../../base -patchesStrategicMerge: -- sriov_numvfs_init.yaml diff --git a/pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go b/pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go index d9cf00deb..ec7309635 100644 --- a/pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go +++ b/pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go @@ -35,6 +35,10 @@ type QatDevicePluginSpec struct { // InitImage is a container image with a script that initialize devices. InitImage string `json:"initImage,omitempty"` + + // ProvisioningConfig is a ConfigMap used to pass the configuration of QAT devices into qat initcontainer. + ProvisioningConfig string `json:"provisioningConfig,omitempty"` + // PreferredAllocationPolicy sets the mode of allocating QAT devices on a node. // See documentation for detailed description of the policies. // +kubebuilder:validation:Enum=balanced;packed diff --git a/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go index 8e0dd446e..a71f6136e 100644 --- a/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go @@ -91,5 +91,28 @@ func (r *QatDevicePlugin) validatePlugin() error { } } + if len(r.Spec.ProvisioningConfig) > 0 { + if len(r.Spec.InitImage) == 0 { + return errors.Errorf("ProvisioningConfig is set with no InitImage") + } + + // check if 4xxxvf is enabled + contains := false + devicesWithCapabilities := map[KernelVfDriver]struct{}{ + "4xxxvf": {}, + } + + for _, kernelVfDriver := range r.Spec.KernelVfDrivers { + if _, ok := devicesWithCapabilities[kernelVfDriver]; ok { + contains = true + break + } + } + + if !contains { + return errors.Errorf("ProvisioningConfig is available only for 4xxx devices") + } + } + return validatePluginImage(r.Spec.Image, "intel-qat-plugin", qatMinVersion) } diff --git a/pkg/controllers/qat/controller.go b/pkg/controllers/qat/controller.go index 759000520..7247fa232 100644 --- a/pkg/controllers/qat/controller.go +++ b/pkg/controllers/qat/controller.go @@ -34,7 +34,10 @@ import ( "github.com/pkg/errors" ) -const ownerKey = ".metadata.controller.qat" +const ( + ownerKey = ".metadata.controller.qat" + initcontainerName = "intel-qat-initcontainer" +) var defaultNodeSelector = deployments.QATPluginDaemonSet().Spec.Template.Spec.NodeSelector @@ -126,6 +129,7 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) ( if ds.Spec.Template.Spec.InitContainers != nil { ds.Spec.Template.Spec.InitContainers = nil ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "sysfs") + ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "qat-config") updated = true } } else { @@ -217,10 +221,20 @@ func setInitContainer(dsSpec *v1.PodSpec, dpSpec devicepluginv1.QatDevicePluginS Image: dpSpec.InitImage, ImagePullPolicy: "IfNotPresent", Name: "init-sriov-numvfs", - Env: []v1.EnvVar{{ - Name: "ENABLED_QAT_PF_PCIIDS", - Value: strings.Join(enablingPfPciIDs, " "), - }}, + Env: []v1.EnvVar{ + { + Name: "ENABLED_QAT_PF_PCIIDS", + Value: strings.Join(enablingPfPciIDs, " "), + }, + { + Name: "NODE_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, SecurityContext: &v1.SecurityContext{ SELinuxOptions: &v1.SELinuxOptions{ Type: "container_device_plugin_init_t", @@ -236,6 +250,29 @@ func setInitContainer(dsSpec *v1.PodSpec, dpSpec devicepluginv1.QatDevicePluginS }, }} addVolumeIfMissing(dsSpec, "sysfs", "/sys", v1.HostPathDirectoryOrCreate) + + mode := int32(0440) + + if dpSpec.ProvisioningConfig != "" { + dsSpec.Volumes = append(dsSpec.Volumes, v1.Volume{ + Name: "qat-config", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{Name: dpSpec.ProvisioningConfig}, + DefaultMode: &mode, + }, + }, + }) + + for i, initcontainer := range dsSpec.InitContainers { + if initcontainer.Name == initcontainerName { + dsSpec.InitContainers[i].VolumeMounts = append(dsSpec.InitContainers[i].VolumeMounts, v1.VolumeMount{ + Name: "qat-config", + MountPath: "/qat-init/conf", + }) + } + } + } } func addVolumeIfMissing(spec *v1.PodSpec, name, path string, hpType v1.HostPathType) { diff --git a/pkg/controllers/qat/controller_test.go b/pkg/controllers/qat/controller_test.go index 3b7a49054..83c3e90cf 100644 --- a/pkg/controllers/qat/controller_test.go +++ b/pkg/controllers/qat/controller_test.go @@ -38,7 +38,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet no := false pluginAnnotations := devicePlugin.ObjectMeta.DeepCopy().Annotations - return &apps.DaemonSet{ + daemonSet := apps.DaemonSet{ TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: "apps/v1", @@ -68,7 +68,17 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet AutomountServiceAccountToken: &no, Containers: []v1.Container{ { - Name: appLabel, + Name: appLabel, + Env: []v1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, Args: getPodArgs(devicePlugin), Image: devicePlugin.Spec.Image, ImagePullPolicy: "IfNotPresent", @@ -140,6 +150,12 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet }, }, } + // add the optional init container + if devicePlugin.Spec.InitImage != "" { + setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec) + } + + return &daemonSet } // Test that QAT daemonset created by using go:embed is diff --git a/test/envtest/qatdeviceplugin_controller_test.go b/test/envtest/qatdeviceplugin_controller_test.go index 626004584..7e09a4079 100644 --- a/test/envtest/qatdeviceplugin_controller_test.go +++ b/test/envtest/qatdeviceplugin_controller_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -92,6 +93,7 @@ var _ = Describe("QatDevicePlugin Controller", func() { By("updating QatDevicePlugin successfully") updatedImage := "updated-qat-testimage" updatedInitImage := "updated-qat-testinitimage" + updatedProvisioningConfig := "updated-qat-provisioningconfig" updatedLogLevel := 2 updatedDpdkDriver := "igb_uio" updatedKernelVfDrivers := "c3xxxvf" @@ -101,6 +103,7 @@ var _ = Describe("QatDevicePlugin Controller", func() { fetched.Spec.Image = updatedImage fetched.Spec.InitImage = updatedInitImage + fetched.Spec.ProvisioningConfig = updatedProvisioningConfig fetched.Spec.LogLevel = updatedLogLevel fetched.Spec.DpdkDriver = updatedDpdkDriver fetched.Spec.KernelVfDrivers = []devicepluginv1.KernelVfDriver{devicepluginv1.KernelVfDriver(updatedKernelVfDrivers)} @@ -131,11 +134,24 @@ var _ = Describe("QatDevicePlugin Controller", func() { "-allocation-policy", updatedPreferredAllocationPolicy, } + mode := int32(0440) + expectedVolume := v1.Volume{ + Name: "qat-config", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{Name: updatedProvisioningConfig}, + DefaultMode: &mode, + }, + }, + } Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs)) Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage)) Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage)) + + Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(expectedVolume)) + Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector)) By("updating QatDevicePlugin with different values successfully") @@ -151,6 +167,7 @@ var _ = Describe("QatDevicePlugin Controller", func() { By("checking DaemonSet is updated with different values successfully") _ = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "intel-qat-plugin"}, ds) Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(0)) + Expect(ds.Spec.Template.Spec.Volumes).ShouldNot(ContainElement(expectedVolume)) Expect(ds.Spec.Template.Spec.NodeSelector).Should(And(HaveLen(1), HaveKeyWithValue("kubernetes.io/arch", "amd64"))) By("deleting QatDevicePlugin successfully")