diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f858f3af9..b83b3df48 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -75,6 +75,7 @@ jobs: - intel-deviceplugin-operator - intel-sgx-plugin - intel-sgx-initcontainer + - intel-dsa-plugin # Demo images - crypto-perf diff --git a/README.md b/README.md index 0533351e6..54f35e5e2 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Table of Contents * [QAT device plugin](#qat-device-plugin) * [VPU device plugin](#vpu-device-plugin) * [SGX device plugin](#sgx-device-plugin) + * [DSA device pugin](#dsa-device-plugin) * [Device Plugins Operator](#device-plugins-operator) * [Demos](#demos) * [Developers](#developers) @@ -163,6 +164,11 @@ operator deployment and NFD is configured to register the SGX EPC memory extende Containers requesting SGX EPC resources in the cluster use `sgx.intel.com/epc` resource which is of type [memory](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory). + +### DSA device plugin + +The [DSA device plugin](cmd/dsa_plugin/README.md) supports acceleration using the Intel Data Streaming accelerator(DSA). + ## Device Plugins Operator Currently the operator has limited support for the QAT, GPU, FPGA and SGX device plugins: diff --git a/build/docker/intel-dsa-plugin.Dockerfile b/build/docker/intel-dsa-plugin.Dockerfile new file mode 100644 index 000000000..54b1dc38e --- /dev/null +++ b/build/docker/intel-dsa-plugin.Dockerfile @@ -0,0 +1,38 @@ +# CLEAR_LINUX_BASE and CLEAR_LINUX_VERSION can be used to make the build +# reproducible by choosing an image by its hash and installing an OS version +# with --version=: +# CLEAR_LINUX_BASE=clearlinux@sha256:b8e5d3b2576eb6d868f8d52e401f678c873264d349e469637f98ee2adf7b33d4 +# CLEAR_LINUX_VERSION="--version=29970" +# +# This is used on release branches before tagging a stable version. +# The master branch defaults to using the latest Clear Linux. +ARG CLEAR_LINUX_BASE=clearlinux/golang:latest + +FROM ${CLEAR_LINUX_BASE} as builder + +ARG CLEAR_LINUX_VERSION= + +RUN swupd update --no-boot-update ${CLEAR_LINUX_VERSION} + +ARG DIR=/intel-device-plugins-for-kubernetes +ARG GO111MODULE=on +WORKDIR $DIR +COPY . . + +RUN mkdir /install_root \ + && swupd os-install \ + ${CLEAR_LINUX_VERSION} \ + --path /install_root \ + --statedir /swupd-state \ + --no-boot-update \ + && rm -rf /install_root/var/lib/swupd/* + +RUN cd cmd/dsa_plugin; GO111MODULE=${GO111MODULE} go install; cd - +RUN chmod a+x /go/bin/dsa_plugin \ + && install -D /go/bin/dsa_plugin /install_root/usr/local/bin/intel_dsa_device_plugin \ + && install -D ${DIR}/LICENSE /install_root/usr/local/share/package-licenses/intel-device-plugins-for-kubernetes/LICENSE \ + && scripts/copy-modules-licenses.sh ./cmd/dsa_plugin /install_root/usr/local/share/ + +FROM scratch as final +COPY --from=builder /install_root / +ENTRYPOINT ["/usr/local/bin/intel_dsa_device_plugin"] diff --git a/cmd/dsa_plugin/README.md b/cmd/dsa_plugin/README.md new file mode 100644 index 000000000..858d9a797 --- /dev/null +++ b/cmd/dsa_plugin/README.md @@ -0,0 +1,130 @@ +# Intel DSA device plugin for Kubernetes + +Table of Contents + +* [Introduction](#introduction) +* [Installation](#installation) + * [Deploy with pre-built container image](#deploy-with-pre-built-container-image) + * [Getting the source code](#getting-the-source-code) + * [Verify node kubelet config](#verify-node-kubelet-config) + * [Deploying as a DaemonSet](#deploying-as-a-daemonset) + * [Build the plugin image](#build-the-plugin-image) + * [Deploy plugin DaemonSet](#deploy-plugin-daemonset) + * [Deploy by hand](#deploy-by-hand) + * [Build the plugin](#build-the-plugin) + * [Run the plugin as administrator](#run-the-plugin-as-administrator) + * [Verify plugin registration](#verify-plugin-registration) + +## Introduction + +The DSA device plugin for Kubernetes supports acceleration using the Intel Data Streaming accelerator(DSA). + +The DSA plugin discovers DSA work queues and presents them as a node resources. + +## Installation + +The following sections detail how to obtain, build, deploy and test the DSA device plugin. + +Examples are provided showing how to deploy the plugin either using a DaemonSet or by hand on a per-node basis. + +### Deploy with pre-built container image + +[Pre-built images](https://hub.docker.com/r/intel/intel-dsa-plugin) +of this component are available on the Docker hub. These images are automatically built and uploaded +to the hub from the latest master branch of this repository. + +Release tagged images of the components are also available on the Docker hub, tagged with their +release version numbers in the format `x.y.z`, corresponding to the branches and releases in this +repository. Thus the easiest way to deploy the plugin in your cluster is to run this command + +```bash +$ kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/dsa_plugin?ref= +daemonset.apps/intel-dsa-plugin created +``` + +Where `` needs to be substituted with the desired git ref, e.g. `master`. + +Nothing else is needed. But if you want to deploy a customized version of the plugin read further. + +### Getting the source code + +```bash +$ export INTEL_DEVICE_PLUGINS_SRC=/path/to/intel-device-plugins-for-kubernetes +$ git clone https://github.com/intel/intel-device-plugins-for-kubernetes ${INTEL_DEVICE_PLUGINS_SRC} +``` + +### Verify node kubelet config + +Every node that will be running the dsa plugin must have the +[kubelet device-plugins](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/) +configured. For each node, check that the kubelet device plugin socket exists: + +```bash +$ ls /var/lib/kubelet/device-plugins/kubelet.sock +/var/lib/kubelet/device-plugins/kubelet.sock +``` + +### Deploying as a DaemonSet + +To deploy the dsa plugin as a daemonset, you first need to build a container image for the +plugin and ensure that is visible to your nodes. + +#### Build the plugin image + +The following will use `docker` to build a local container image called +`intel/intel-dsa-plugin` with the tag `devel`. + +The image build tool can be changed from the default `docker` by setting the `BUILDER` argument +to the [`Makefile`](Makefile). + +```bash +$ cd ${INTEL_DEVICE_PLUGINS_SRC} +$ make intel-dsa-plugin +... +Successfully tagged intel/intel-dsa-plugin:devel +``` + +#### Deploy plugin DaemonSet + +You can then use the [example DaemonSet YAML](/deployments/dsa_plugin/base/intel-dsa-plugin.yaml) +file provided to deploy the plugin. The default kustomization that deploys the YAML as is: + +```bash +$ kubectl apply -k deployments/dsa_plugin +daemonset.apps/intel-dsa-plugin created +``` + +### Deploy by hand + +For development purposes, it is sometimes convenient to deploy the plugin 'by hand' on a node. +In this case, you do not need to build the complete container image, and can build just the plugin. + +#### Build the plugin + +First we build the plugin: + +```bash +$ cd ${INTEL_DEVICE_PLUGINS_SRC} +$ make dsa_plugin +``` + +#### Run the plugin as administrator + +Now we can run the plugin directly on the node: + +```bash +$ sudo -E ${INTEL_DEVICE_PLUGINS_SRC}/cmd/dsa_plugin/dsa_plugin +device-plugin registered +``` + +### Verify plugin registration + +You can verify the plugin has been registered with the expected nodes by searching for the relevant +resource allocation status on the nodes: + +```bash +$ kubectl get nodes -o=jsonpath="{range .items[*]}{.metadata.name}{'\n'}{' i915: '}{.status.allocatable.dsa\.intel\.com/*}{'\n'}" +master + dsa.intel.com/wq-user-dedicated: 1 + dsa.intel.com/wq-user-shared: 1 +``` diff --git a/cmd/dsa_plugin/dsa_plugin.go b/cmd/dsa_plugin/dsa_plugin.go new file mode 100644 index 000000000..23c1f6ac4 --- /dev/null +++ b/cmd/dsa_plugin/dsa_plugin.go @@ -0,0 +1,55 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "os" + + dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/idxd" + + "k8s.io/klog" +) + +const ( + // Device plugin settings. + namespace = "dsa.intel.com" + // SysFS directory. + sysfsDir = "/sys/bus/dsa/devices" + // Device directories. + devDir = "/dev/dsa" + // Glob pattern for the state sysfs entry. + statePattern = "/sys/bus/dsa/devices/dsa*/wq*/state" +) + +func main() { + var sharedDevNum int + + flag.IntVar(&sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same work queue") + flag.Parse() + + if sharedDevNum < 1 { + klog.Warning("The number of containers sharing the same work queue must be greater than zero") + os.Exit(1) + } + + plugin := idxd.NewDevicePlugin(sysfsDir, statePattern, devDir, sharedDevNum) + if plugin == nil { + klog.Fatal("Cannot create device plugin, please check above error messages.") + } + manager := dpapi.NewManager(namespace, plugin) + manager.Run() +} diff --git a/deployments/dsa_plugin/base/intel-dsa-plugin.yaml b/deployments/dsa_plugin/base/intel-dsa-plugin.yaml new file mode 100644 index 000000000..8b3aa910d --- /dev/null +++ b/deployments/dsa_plugin/base/intel-dsa-plugin.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-dsa-plugin + labels: + app: intel-dsa-plugin +spec: + selector: + matchLabels: + app: intel-dsa-plugin + template: + metadata: + labels: + app: intel-dsa-plugin + spec: + containers: + - name: intel-dsa-plugin + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: intel/intel-dsa-plugin:devel + imagePullPolicy: IfNotPresent + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - name: devfs + mountPath: /dev/dsa + readOnly: true + - name: chardevs + mountPath: /dev/char + readOnly: true + - name: sysfs + mountPath: /sys/bus/dsa + readOnly: true + - name: kubeletsockets + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: devfs + hostPath: + path: /dev/dsa + - name: chardevs + hostPath: + path: /dev/char + - name: sysfs + hostPath: + path: /sys/bus/dsa + - name: kubeletsockets + hostPath: + path: /var/lib/kubelet/device-plugins + nodeSelector: + kubernetes.io/arch: amd64 diff --git a/deployments/dsa_plugin/base/kustomization.yaml b/deployments/dsa_plugin/base/kustomization.yaml new file mode 100644 index 000000000..03f141f2e --- /dev/null +++ b/deployments/dsa_plugin/base/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - intel-dsa-plugin.yaml diff --git a/deployments/dsa_plugin/kustomization.yaml b/deployments/dsa_plugin/kustomization.yaml new file mode 100644 index 000000000..f191f3aae --- /dev/null +++ b/deployments/dsa_plugin/kustomization.yaml @@ -0,0 +1,2 @@ +bases: + - base diff --git a/deployments/dsa_plugin/overlays/namespace_kube-system/add-namespace-kube-system.yaml b/deployments/dsa_plugin/overlays/namespace_kube-system/add-namespace-kube-system.yaml new file mode 100644 index 000000000..1a4ae4769 --- /dev/null +++ b/deployments/dsa_plugin/overlays/namespace_kube-system/add-namespace-kube-system.yaml @@ -0,0 +1,5 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-dsa-plugin + namespace: kube-system diff --git a/deployments/dsa_plugin/overlays/namespace_kube-system/kustomization.yaml b/deployments/dsa_plugin/overlays/namespace_kube-system/kustomization.yaml new file mode 100644 index 000000000..7d41bcfbe --- /dev/null +++ b/deployments/dsa_plugin/overlays/namespace_kube-system/kustomization.yaml @@ -0,0 +1,4 @@ +bases: + - ../../base +patches: + - add-namespace-kube-system.yaml diff --git a/pkg/idxd/plugin.go b/pkg/idxd/plugin.go new file mode 100644 index 000000000..9b909ec09 --- /dev/null +++ b/pkg/idxd/plugin.go @@ -0,0 +1,202 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package idxd + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "syscall" + "time" + + dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" + "github.com/pkg/errors" + "golang.org/x/sys/unix" + "k8s.io/klog" + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" +) + +const ( + // Character devices directory. + charDevDir = "/dev/char" + // Frequency of device scans. + scanFrequency = 5 * time.Second +) + +// getDevNodesFunc type allows overriding filesystem APIs (os.Stat, stat.Sys, etc) in tests. +type getDevNodesFunc func(devDir, charDevDir, wqName string) ([]pluginapi.DeviceSpec, error) + +// DevicePlugin defines properties of the idxd device plugin. +type DevicePlugin struct { + sysfsDir string + statePattern string + devDir string + charDevDir string + sharedDevNum int + scanTicker *time.Ticker + scanDone chan bool + getDevNodes getDevNodesFunc +} + +// NewDevicePlugin creates DevicePlugin. +func NewDevicePlugin(sysfsDir, statePattern, devDir string, sharedDevNum int) *DevicePlugin { + return &DevicePlugin{ + sysfsDir: sysfsDir, + statePattern: statePattern, + devDir: devDir, + charDevDir: charDevDir, + sharedDevNum: sharedDevNum, + scanTicker: time.NewTicker(scanFrequency), + scanDone: make(chan bool, 1), + getDevNodes: getDevNodes, + } +} + +// Scan discovers devices and reports them to the upper level API. +func (dp *DevicePlugin) Scan(notifier dpapi.Notifier) error { + defer dp.scanTicker.Stop() + for { + devTree, err := dp.scan() + if err != nil { + return err + } + + notifier.Notify(devTree) + + select { + case <-dp.scanDone: + return nil + case <-dp.scanTicker.C: + } + } +} + +func readFile(fpath string) (string, error) { + data, err := ioutil.ReadFile(fpath) + if err != nil { + return "", errors.WithStack(err) + } + return strings.TrimSpace(string(data)), nil +} + +// getDevNodes collects device nodes that belong to working queue. +func getDevNodes(devDir, charDevDir, wqName string) ([]pluginapi.DeviceSpec, error) { + // check if /dev/dsa/ device node exists + devPath := path.Join(devDir, wqName) + stat, err := os.Stat(devPath) + if err != nil { + return nil, errors.WithStack(err) + } + // Check if it's a character device + if stat.Mode()&os.ModeCharDevice == 0 { + return nil, errors.Errorf("%s is not a character device", devPath) + } + + // get /dev/char/: symlink for the device node + // as libaccel-config requires it + rdev := stat.Sys().(*syscall.Stat_t).Rdev + charDevPath := path.Join(charDevDir, fmt.Sprintf("%d:%d", unix.Major(rdev), unix.Minor(rdev))) + stat, err = os.Lstat(charDevPath) + if err != nil { + return nil, errors.WithStack(err) + } + if stat.Mode()&os.ModeSymlink == 0 { + return nil, errors.Errorf("%s is not a symlink", charDevPath) + } + // Check if symlink points to the correct device node + destPath, err := filepath.EvalSymlinks(charDevPath) + if err != nil { + return nil, errors.WithStack(err) + } + if destPath != devPath { + return nil, errors.Errorf("%s points to %s instead of device node %s", charDevPath, destPath, devPath) + } + + // report device node and /dev/char/: symlink + // as libaccel-config works with a symlink + return []pluginapi.DeviceSpec{ + { + HostPath: devPath, + ContainerPath: devPath, + Permissions: "rw", + }, + { + HostPath: charDevPath, + ContainerPath: charDevPath, + Permissions: "rw", + }}, nil +} + +// scan collects devices by scanning sysfs and devfs entries. +func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) { + // scan sysfs tree + matches, err := filepath.Glob(dp.statePattern) + if err != nil { + return nil, errors.WithStack(err) + } + + devTree := dpapi.NewDeviceTree() + + for _, fpath := range matches { + // Read queue state entry + wqState, err := readFile(fpath) + if err != nil { + return nil, err + } + + if wqState != "enabled" { + continue + } + + // Read queue mode + queueDir := filepath.Dir(fpath) + wqMode, err := readFile(path.Join(queueDir, "mode")) + if err != nil { + return nil, err + } + + // Read queue type + wqType, err := readFile(path.Join(queueDir, "type")) + if err != nil { + return nil, err + } + + wqName := filepath.Base(queueDir) + devNodes := []pluginapi.DeviceSpec{} + + if wqType == "user" { + devNodes, err = dp.getDevNodes(dp.devDir, dp.charDevDir, wqName) + if err != nil { + return nil, err + } + } + + amount := dp.sharedDevNum + if wqMode != "shared" { + amount = 1 + } + klog.V(4).Infof("%s: amount: %d, type: %s, mode: %s, nodes: %+v", wqName, amount, wqType, wqMode, devNodes) + for i := 0; i < amount; i++ { + deviceType := fmt.Sprintf("wq-%s-%s", wqType, wqMode) + deviceID := fmt.Sprintf("%s-%s-%d", deviceType, wqName, i) + devTree.AddDevice(deviceType, deviceID, dpapi.NewDeviceInfo(pluginapi.Healthy, devNodes, nil, nil)) + } + } + + return devTree, nil +} diff --git a/pkg/idxd/plugin_test.go b/pkg/idxd/plugin_test.go new file mode 100644 index 000000000..671fec577 --- /dev/null +++ b/pkg/idxd/plugin_test.go @@ -0,0 +1,256 @@ +// Copyright 2017 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package idxd + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "path" + "testing" + + dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" +) + +const ( + dsaMajor int = 375 +) + +func getFakeDevNodes(devDir, charDevDir, wqName string) ([]pluginapi.DeviceSpec, error) { + devPath := path.Join(devDir, wqName) + var devNum int + var queueNum int + fmt.Sscanf(wqName, "wq%d.%d", &devNum, &queueNum) + charDevPath := path.Join(charDevDir, fmt.Sprintf("%d:%d", dsaMajor, devNum*10+queueNum)) + + return []pluginapi.DeviceSpec{ + { + HostPath: devPath, + ContainerPath: devPath, + Permissions: "rw", + }, + { + HostPath: charDevPath, + ContainerPath: charDevPath, + Permissions: "rw", + }}, nil +} + +func init() { + _ = flag.Set("v", "4") //Enable debug output +} + +// fakeNotifier implements Notifier interface. +type fakeNotifier struct { + scanDone chan bool + deviceTree dpapi.DeviceTree +} + +// Notify stops plugin Scan. +func (n *fakeNotifier) Notify(deviceTree dpapi.DeviceTree) { + n.deviceTree = deviceTree + n.scanDone <- true +} + +type testCase struct { + name string + sysfsDirs []string + sysfsFiles map[string][]byte + sharedDevNum int + expectedResult map[string]int + expectedError bool +} + +func TestScan(t *testing.T) { + root, err := ioutil.TempDir("", "test_idxd_device_plugin") + if err != nil { + t.Fatalf("can't create temporary directory: %+v", err) + } + + defer os.RemoveAll(root) + + sysfs := path.Join(root, "sys/bus/dsa/devices") + statePattern := path.Join(sysfs, "dsa*/wq*/state") + + tcases := []testCase{ + { + name: "no sysfs mounted", + }, + { + name: "all queues are disabled", + sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa0/wq0.2", "dsa0/wq0.3"}, + sysfsFiles: map[string][]byte{ + "dsa0/wq0.0/state": []byte(""), + "dsa0/wq0.1/state": []byte(""), + "dsa0/wq0.2/state": []byte(""), + "dsa0/wq0.3/state": []byte(""), + }, + }, + { + name: "invalid: mode entry doesn't exist", + sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa0/wq0.2", "dsa0/wq0.3"}, + sysfsFiles: map[string][]byte{ + "dsa0/wq0.0/state": []byte("enabled"), + "dsa0/wq0.1/state": []byte("enabled"), + "dsa0/wq0.2/state": []byte(""), + "dsa0/wq0.3/state": []byte(""), + }, + expectedError: true, + }, + { + name: "invalid: type entry doesn't exist", + sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa0/wq0.2", "dsa0/wq0.3"}, + sysfsFiles: map[string][]byte{ + "dsa0/wq0.0/state": []byte("enabled"), + "dsa0/wq0.1/state": []byte("enabled"), + "dsa0/wq0.2/state": []byte(""), + "dsa0/wq0.3/state": []byte(""), + "dsa0/wq0.0/mode": []byte("dedicated"), + "dsa0/wq0.1/mode": []byte("dedicated"), + }, + expectedError: true, + }, + { + name: "valid: two dedicated user queues", + sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa0/wq0.2", "dsa0/wq0.3"}, + sysfsFiles: map[string][]byte{ + "dsa0/wq0.0/state": []byte("enabled"), + "dsa0/wq0.1/state": []byte("enabled"), + "dsa0/wq0.2/state": []byte(""), + "dsa0/wq0.3/state": []byte(""), + "dsa0/wq0.0/mode": []byte("dedicated"), + "dsa0/wq0.1/mode": []byte("dedicated"), + "dsa0/wq0.0/type": []byte("user"), + "dsa0/wq0.1/type": []byte("user"), + }, + expectedResult: map[string]int{ + "wq-user-dedicated": 2, + }, + }, + { + name: "valid: two shared user queues x 10", + sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa0/wq0.2", "dsa0/wq0.3"}, + sysfsFiles: map[string][]byte{ + "dsa0/wq0.0/state": []byte("enabled"), + "dsa0/wq0.1/state": []byte("enabled"), + "dsa0/wq0.2/state": []byte(""), + "dsa0/wq0.3/state": []byte(""), + "dsa0/wq0.0/mode": []byte("shared"), + "dsa0/wq0.1/mode": []byte("shared"), + "dsa0/wq0.0/type": []byte("user"), + "dsa0/wq0.1/type": []byte("user"), + }, + sharedDevNum: 10, + expectedResult: map[string]int{ + "wq-user-shared": 20, + }, + }, + { + name: "valid: all types of queues", + sysfsDirs: []string{"dsa0/wq0.0", "dsa0/wq0.1", "dsa1/wq1.0", "dsa1/wq1.1", "dsa1/wq1.2", "dsa1/wq1.3"}, + sysfsFiles: map[string][]byte{ + //device 0 + "dsa0/wq0.0/state": []byte("enabled"), + "dsa0/wq0.1/state": []byte("enabled"), + "dsa0/wq0.0/mode": []byte("shared"), + "dsa0/wq0.1/mode": []byte("dedicated"), + "dsa0/wq0.0/type": []byte("user"), + "dsa0/wq0.1/type": []byte("kernel"), + //device 1 + "dsa1/wq1.0/state": []byte("enabled"), + "dsa1/wq1.1/state": []byte("enabled"), + "dsa1/wq1.2/state": []byte("enabled"), + "dsa1/wq1.3/state": []byte("enabled"), + "dsa1/wq1.0/mode": []byte("shared"), + "dsa1/wq1.1/mode": []byte("dedicated"), + "dsa1/wq1.2/mode": []byte("shared"), + "dsa1/wq1.3/mode": []byte("dedicated"), + "dsa1/wq1.0/type": []byte("mdev"), + "dsa1/wq1.1/type": []byte("mdev"), + "dsa1/wq1.2/type": []byte("mdev"), + "dsa1/wq1.3/type": []byte("user"), + }, + sharedDevNum: 10, + expectedResult: map[string]int{ + "wq-user-shared": 10, + "wq-kernel-dedicated": 1, + "wq-mdev-shared": 20, + "wq-user-dedicated": 1, + "wq-mdev-dedicated": 1, + }, + }, + } + + for _, tc := range tcases { + t.Run(tc.name, genTest(sysfs, statePattern, tc)) + } +} + +// generate test to decrease cyclomatic complexity. +func genTest(sysfs, statePattern string, tc testCase) func(t *testing.T) { + return func(t *testing.T) { + for _, sysfsdir := range tc.sysfsDirs { + if err := os.MkdirAll(path.Join(sysfs, sysfsdir), 0750); err != nil { + t.Fatalf("Failed to create fake sysfs directory: %+v", err) + } + } + for filename, body := range tc.sysfsFiles { + if err := ioutil.WriteFile(path.Join(sysfs, filename), body, 0600); err != nil { + t.Fatalf("Failed to create fake sysfs entry: %+v", err) + } + } + + plugin := NewDevicePlugin(sysfs, statePattern, "", tc.sharedDevNum) + plugin.getDevNodes = getFakeDevNodes + + notifier := &fakeNotifier{ + scanDone: plugin.scanDone, + } + + err := plugin.Scan(notifier) + if !tc.expectedError && err != nil { + t.Errorf("unexpected error: %+v", err) + } + if tc.expectedError && err == nil { + t.Errorf("unexpected success") + } + if err := checkDeviceTree(notifier.deviceTree, tc.expectedResult, tc.expectedError); err != nil { + t.Error(err) + } + } +} + +// checkDeviceTree checks discovered device types and number of discovered devices. +func checkDeviceTree(deviceTree dpapi.DeviceTree, expectedResult map[string]int, expectedError bool) error { + if !expectedError && deviceTree != nil { + for key := range deviceTree { + val, ok := expectedResult[key] + if !ok { + return fmt.Errorf("unexpected device type: %s", key) + } + numberDev := len(deviceTree[key]) + if numberDev != val { + return fmt.Errorf("%s: unexpected number of devices: %d, expected: %d", key, numberDev, val) + } + delete(expectedResult, key) + } + if len(expectedResult) > 0 { + return fmt.Errorf("missing expected result(s): %+v", expectedResult) + } + } + return nil +}