Add macvtap device plugin

This commit adds the initial implementation of the macvtap device
plugin. Includes manifests & templates for deployment, as well as
some basic tests.

Further changes are expected as the macvtap cni is integrated.
RBAC and more thorough testing pending for a later patch.

.gitignore

@@ -3,6 +3,7 @@ goimports-check
 goimports-format
 whitespace-check
 whitespace-format
+vet
 
 # Local kubevirtci cluster
 _kubevirtci/

Makefile

@@ -6,7 +6,8 @@ TARGETS = \
 	goimports-format \
 	goimports-check \
 	whitespace-format \
-	whitespace-check
+	whitespace-check \
+	vet
 
 # Make does not offer a recursive wildcard function, so here's one:
 rwildcard=$(wildcard $1$2) $(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2))
@@ -15,9 +16,11 @@ rwildcard=$(wildcard $1$2) $(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2))
 directories := $(filter-out ./ ./vendor/ ,$(sort $(dir $(wildcard ./*/))))
 all_sources=$(call rwildcard,$(directories),*) $(filter-out $(TARGETS), $(wildcard *))
 
+.ONESHELL:
+
 all: format check
 
-check: goimports-check whitespace-check test/unit
+check: goimports-check whitespace-check vet test/unit
 
 format: goimports-format whitespace-format
 
@@ -37,8 +40,12 @@ whitespace-format: $(all_sources)
 	go run ./vendor/golang.org/x/tools/cmd/goimports -w ./pkg ./cmd ./test
 	touch $@
 
+vet: $(all_sources)
+	go vet ./pkg/... ./cmd/... ./test/...
+	touch $@
+
 docker-build:
-	docker build -t ${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG} ./cmd
+	docker build -t ${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG} -f ./cmd/Dockerfile .
 
 docker-push:
 	docker push ${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}
@@ -56,7 +63,11 @@ test/e2e:
 	./hack/functest.sh
 
 test/unit:
-	go test ./cmd/... ./pkg/... -v --ginkgo.v
+	@if ! [ "$$(id -u)" = 0 ]; then
+		@echo "You are not root, run this target as root please"
+		exit 1
+	fi
+	go test ./cmd/... ./pkg/... -v -logtostderr --ginkgo.v
 
 manifests:
 	IMAGE_REGISTRY=$(IMAGE_REGISTRY) IMAGE_NAME=$(IMAGE_NAME) IMAGE_TAG=$(IMAGE_TAG) ./hack/generate-manifests.sh

cmd/Dockerfile

@@ -1 +1,11 @@
 # Multi-stage dockerfile building a container image with both binaries included
+
+FROM golang:1.13 as builder
+ENV GOPATH=/go
+WORKDIR /go/src/github.com/kubevirt/macvtap-cni
+COPY . .
+RUN GOOS=linux CGO_ENABLED=0 go build -o /macvtap github.com/kubevirt/macvtap-cni/cmd/deviceplugin
+
+FROM registry.access.redhat.com/ubi8/ubi-minimal
+COPY --from=builder /macvtap /macvtap
+ENTRYPOINT [ "./macvtap", "-v", "3", "-logtostderr"]

cmd/deviceplugin/macvtap.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	"github.com/golang/glog"
+	"github.com/kubevirt/device-plugin-manager/pkg/dpm"
+	macvtap "github.com/kubevirt/macvtap-cni/pkg/deviceplugin"
+)
+
+func main() {
+	flag.Parse()
+
+	_, configDefined := os.LookupEnv(macvtap.ConfigEnvironmentVariable)
+	if !configDefined {
+		glog.Exitf("%s environment variable must be set", macvtap.ConfigEnvironmentVariable)
+	}
+
+	manager := dpm.NewManager(macvtap.MacvtapLister{})
+	manager.Run()
+}

examples/macvtap-deviceplugin-config.yaml

@@ -0,0 +1,12 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: macvtap-deviceplugin-config
+data:
+  DP_MACVTAP_CONF: >-
+    [ {
+        "name" : "eth0",
+        "master" : "eth0",
+        "mode": "bridge",
+        "capacity" : 50
+     } ]

go.mod

@@ -3,10 +3,21 @@ module github.com/kubevirt/macvtap-cni
 go 1.13
 
 require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/client9/misspell v0.3.4 // indirect
+	github.com/containernetworking/plugins v0.8.5
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/kubevirt/device-plugin-manager v1.14.0
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.8.1
+	github.com/vishvananda/netlink v1.1.0
+	golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect
+	golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc
 	golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9
-	k8s.io/client-go v0.0.0-00010101000000-000000000000
+	google.golang.org/grpc v1.26.0
+	honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect
+	k8s.io/client-go v0.0.0
+	k8s.io/kubernetes v0.0.0-00010101000000-000000000000
 )
 
 // Pinned to kubernetes-1.15.4
@@ -28,6 +39,7 @@ replace (
 	k8s.io/kube-proxy => k8s.io/kube-proxy v0.0.0-20190918202429-08c8357f8e2d
 	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.0.0-20190918202713-c34a54b3ec8e
 	k8s.io/kubelet => k8s.io/kubelet v0.0.0-20190918202550-958285cf3eef
+	k8s.io/kubernetes => k8s.io/kubernetes v1.15.4
 	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.0.0-20190918203421-225f0541b3ea
 	k8s.io/metrics => k8s.io/metrics v0.0.0-20190918202012-3c1ca76f5bda
 	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.0.0-20190918201353-5cc279503896

manifests/macvtap.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: macvtap-cni
+spec:
+  selector:
+    matchLabels:
+      name: macvtap-cni
+  template:
+    metadata:
+      labels:
+        name: macvtap-cni
+    spec:
+      hostNetwork: true
+      hostPID: true
+      containers:
+      - name: macvtap-cni
+        image: quay.io/kubevirt/macvtap-cni:latest
+        securityContext:
+          privileged: true
+        envFrom:
+        - configMapRef:
+            name: macvtap-deviceplugin-config
+        volumeMounts:
+          - name: deviceplugin
+            mountPath: /var/lib/kubelet/device-plugins
+      volumes:
+        - name: deviceplugin
+          hostPath:
+            path: /var/lib/kubelet/device-plugins

pkg/deviceplugin/deviceplugin_suite_test.go

@@ -0,0 +1,13 @@
+package deviceplugin_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestDevicePlugin(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Deviceplugin Suite")
+}

pkg/deviceplugin/lister.go

@@ -0,0 +1,74 @@
+package deviceplugin
+
+import (
+	"encoding/json"
+	"os"
+
+	"github.com/golang/glog"
+	"github.com/kubevirt/device-plugin-manager/pkg/dpm"
+)
+
+const (
+	resourceNamespace         = "macvtap.network.kubevirt.io"
+	ConfigEnvironmentVariable = "DP_MACVTAP_CONF"
+)
+
+type MacvtapConfig struct {
+	Name     string `json:"name"`
+	Master   string `json:"master"`
+	Mode     string `json:"mode"`
+	Capacity int    `json:"capacity"`
+}
+
+type MacvtapLister struct {
+}
+
+func (ml MacvtapLister) GetResourceNamespace() string {
+	return resourceNamespace
+}
+
+func readConfig() (map[string]MacvtapConfig, error) {
+	var config []MacvtapConfig
+	configMap := make(map[string]MacvtapConfig)
+
+	configEnv := os.Getenv(ConfigEnvironmentVariable)
+	err := json.Unmarshal([]byte(configEnv), &config)
+	if err != nil {
+		return configMap, err
+	}
+
+	for _, macvtapConfig := range config {
+		configMap[macvtapConfig.Name] = macvtapConfig
+	}
+
+	return configMap, nil
+}
+
+func (ml MacvtapLister) Discover(pluginListCh chan dpm.PluginNameList) {
+	var plugins = make(dpm.PluginNameList, 0)
+
+	config, err := readConfig()
+	if err != nil {
+		glog.Errorf("Error reading config: %v", err)
+		return
+	}
+
+	glog.V(3).Infof("Read configuration %+v", config)
+
+	for _, macvtapConfig := range config {
+		plugins = append(plugins, macvtapConfig.Name)
+	}
+
+	pluginListCh <- plugins
+}
+
+func (ml MacvtapLister) NewPlugin(name string) dpm.PluginInterface {
+	config, _ := readConfig()
+	glog.V(3).Infof("Creating device plugin with config %+v", config[name])
+	return NewMacvtapDevicePlugin(
+		config[name].Name,
+		config[name].Master,
+		config[name].Mode,
+		config[name].Capacity,
+	)
+}

pkg/deviceplugin/plugin.go

@@ -0,0 +1,133 @@
+package deviceplugin
+
+import (
+	"fmt"
+
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+
+	"github.com/kubevirt/macvtap-cni/pkg/util"
+)
+
+const (
+	tapPath         = "/dev/tap"
+	suffix          = "Mvp"
+	defaultCapacity = 100
+)
+
+type MacvtapDevicePlugin struct {
+	Name        string
+	Master      string
+	Mode        string
+	Capacity    int
+	stopWatcher chan struct{}
+}
+
+func NewMacvtapDevicePlugin(name string, master string, mode string, capacity int) *MacvtapDevicePlugin {
+	return &MacvtapDevicePlugin{
+		Name:        name,
+		Master:      master,
+		Mode:        mode,
+		Capacity:    capacity,
+		stopWatcher: make(chan struct{}),
+	}
+}
+
+func (mdp *MacvtapDevicePlugin) generateMacvtapDevices() []*pluginapi.Device {
+	var macvtapDevs []*pluginapi.Device
+
+	var capacity = mdp.Capacity
+	if capacity <= 0 {
+		capacity = defaultCapacity
+	}
+
+	for i := 0; i < capacity; i++ {
+		name := fmt.Sprint(mdp.Name, suffix, i)
+		macvtapDevs = append(macvtapDevs, &pluginapi.Device{
+			ID:     name,
+			Health: pluginapi.Healthy,
+		})
+	}
+	return macvtapDevs
+}
+
+func masterExists(master string) bool {
+	if err := util.LinkExists(master); err != nil {
+		glog.Warningf("Master %s not found: %v", master, err)
+		return false
+	}
+	return true
+}
+
+func (mdp *MacvtapDevicePlugin) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlugin_ListAndWatchServer) error {
+	masterDevs := mdp.generateMacvtapDevices()
+	noMasterDevs := make([]*pluginapi.Device, 0)
+	emitResponse := func(masterExists bool) {
+		if masterExists {
+			glog.V(3).Info("Master exists, sending ListAndWatch response with available devices")
+			s.Send(&pluginapi.ListAndWatchResponse{Devices: masterDevs})
+		} else {
+			glog.V(3).Info("Master does not exist, sending ListAndWatch response with no devices")
+			s.Send(&pluginapi.ListAndWatchResponse{Devices: noMasterDevs})
+		}
+	}
+
+	didMasterExist := false
+	onMasterEvent := func() {
+		doesMasterExist := masterExists(mdp.Master)
+		if didMasterExist != doesMasterExist {
+			emitResponse(doesMasterExist)
+			didMasterExist = doesMasterExist
+		}
+	}
+
+	util.OnLinkEvent(
+		mdp.Master,
+		onMasterEvent,
+		mdp.stopWatcher,
+		func(err error) {
+			glog.Error(err)
+		})
+
+	return nil
+}
+
+func (mdp *MacvtapDevicePlugin) Allocate(ctx context.Context, r *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
+	var response pluginapi.AllocateResponse
+
+	for _, req := range r.ContainerRequests {
+		var devices []*pluginapi.DeviceSpec
+		for _, name := range req.DevicesIDs {
+			dev := new(pluginapi.DeviceSpec)
+			index, err := util.RecreateMacvtap(name, mdp.Master, mdp.Mode)
+			if err != nil {
+				return nil, err
+			}
+			devPath := fmt.Sprint(tapPath, index)
+			dev.HostPath = devPath
+			dev.ContainerPath = devPath
+			dev.Permissions = "rw"
+			devices = append(devices, dev)
+		}
+
+		response.ContainerResponses = append(response.ContainerResponses, &pluginapi.ContainerAllocateResponse{
+			Devices: devices,
+		})
+	}
+
+	return &response, nil
+}
+
+func (mdp *MacvtapDevicePlugin) PreStartContainer(context.Context, *pluginapi.PreStartContainerRequest) (*pluginapi.PreStartContainerResponse, error) {
+	return nil, nil
+}
+
+func (mdp *MacvtapDevicePlugin) GetDevicePluginOptions(context.Context, *pluginapi.Empty) (*pluginapi.DevicePluginOptions, error) {
+	return nil, nil
+}
+
+func (mdp *MacvtapDevicePlugin) Stop() error {
+	close(mdp.stopWatcher)
+	return nil
+}