Merge pull request #1733 from davidvossel/csi-cleanup-v1

KubeVirt CSI Driver Integration
openshift · Oct 18, 2022 · bb64fb1 · bb64fb1
2 parents 1f70f40 + 420b758
commit bb64fb1
Show file tree

Hide file tree

Showing 15 changed files with 998 additions and 0 deletions.
diff --git a/cmd/install/assets/hypershift_operator.go b/cmd/install/assets/hypershift_operator.go
@@ -835,6 +835,16 @@ func (o HyperShiftOperatorClusterRole) Build() *rbacv1.ClusterRole {
 				Resources: []string{"virtualmachineinstances", "virtualmachines"},
 				Verbs:     []string{"*"},
 			},
+			{ // This allows the kubevirt csi driver to hotplug volumes to KubeVirt VMs.
+				APIGroups: []string{"subresources.kubevirt.io"},
+				Resources: []string{"virtualmachineinstances/addvolume", "virtualmachineinstances/removevolume"},
+				Verbs:     []string{"*"},
+			},
+			{ // This allows the kubevirt csi driver to mirror guest PVCs to the mgmt/infra cluster
+				APIGroups: []string{"cdi.kubevirt.io"},
+				Resources: []string{"datavolumes"},
+				Verbs:     []string{"*"},
+			},
 			{ // This allows hypershift operator to grant RBAC permissions for agents, clusterDeployments and agentClusterInstalls to the capi-provider-agent
 				APIGroups: []string{"agent-install.openshift.io"},
 				Resources: []string{"agents"},

diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/controller.yaml b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/controller.yaml
@@ -0,0 +1,109 @@
+# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: kubevirt-csi-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kubevirt-csi-driver
+  template:
+    metadata:
+      labels:
+        app: kubevirt-csi-driver
+    spec:
+      serviceAccount: kubevirt-csi
+      priorityClassName: hypershift-control-plane
+      containers:
+        - name: csi-driver
+          imagePullPolicy: Always
+          image: quay.io/dvossel/kubevirt-csi-driver:latest
+          args:
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--infra-cluster-namespace=$(INFRACLUSTER_NAMESPACE)"
+            - "--tenant-cluster-kubeconfig=/var/run/secrets/tenantcluster/kubeconfig"
+            - "--infra-cluster-labels=$(INFRACLUSTER_LABELS)"
+            - "--run-node-service=false"
+            - "--run-controller-service=true"
+            - --v=5
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: INFRACLUSTER_NAMESPACE
+              valueFrom:
+                configMapKeyRef:
+                  name: driver-config
+                  key: infraClusterNamespace
+            - name: INFRACLUSTER_LABELS
+              valueFrom:
+                configMapKeyRef:
+                  name: driver-config
+                  key: infraClusterLabels
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: tenantcluster
+              mountPath: "/var/run/secrets/tenantcluster"
+          resources:
+            requests:
+              memory: 50Mi
+              cpu: 10m
+        - name: csi-provisioner
+          image: quay.io/openshift/origin-csi-external-provisioner:latest
+          args:
+            - --csi-address=$(ADDRESS)
+            - --default-fstype=ext4
+            - --v=5
+            - "--kubeconfig=/var/run/secrets/tenantcluster/kubeconfig"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: tenantcluster
+              mountPath: "/var/run/secrets/tenantcluster"
+        - name: csi-attacher
+          image: quay.io/openshift/origin-csi-external-attacher:latest
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=5
+            - "--kubeconfig=/var/run/secrets/tenantcluster/kubeconfig"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: tenantcluster
+              mountPath: "/var/run/secrets/tenantcluster"
+          resources:
+            requests:
+              memory: 50Mi
+              cpu: 10m
+        - name: csi-liveness-probe
+          image: quay.io/openshift/origin-csi-livenessprobe:latest
+          args:
+            - --csi-address=/csi/csi.sock
+            - --probe-timeout=3s
+            - --health-port=10301
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: tenantcluster
+              mountPath: "/var/run/secrets/tenantcluster"
+          resources:
+            requests:
+              memory: 50Mi
+              cpu: 10m
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: tenantcluster
+          secret:
+            secretName: kubevirt-csi-controller-tenant-kubeconfig
diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/daemonset.yaml b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/daemonset.yaml
@@ -0,0 +1,123 @@
+# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: kubevirt-csi-node
+spec:
+  selector:
+    matchLabels:
+      app: kubevirt-csi-driver
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: kubevirt-csi-driver
+    spec:
+      serviceAccount: kubevirt-csi-node-sa
+      priorityClassName: system-node-critical
+      tolerations:
+        - operator: Exists
+      containers:
+        - name: csi-driver
+          securityContext:
+            privileged: true
+            allowPrivilegeEscalation: true
+          imagePullPolicy: Always
+          image: quay.io/dvossel/kubevirt-csi-driver:latest
+          args:
+            - "--endpoint=unix:/csi/csi.sock"
+            - "--node-name=$(KUBE_NODE_NAME)"
+            - "--run-node-service=true"
+            - "--run-controller-service=false"
+            - --v=5
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: device-dir
+              mountPath: /dev
+            - name: udev
+              mountPath: /run/udev
+          ports:
+            - name: healthz
+              containerPort: 10300
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+          resources:
+            requests:
+              memory: 50Mi
+              cpu: 10m
+        - name: csi-node-driver-registrar
+          securityContext:
+            privileged: true
+          image: quay.io/openshift/origin-csi-node-driver-registrar:latest
+          args:
+            - --csi-address=$(ADDRESS)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --v=5
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/csi.kubevirt.io-reg.sock /csi/csi.sock"]
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: /var/lib/kubelet/plugins/csi.kubevirt.io/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+          resources:
+            requests:
+              memory: 20Mi
+              cpu: 5m
+        - name: csi-liveness-probe
+          image: quay.io/openshift/origin-csi-livenessprobe:latest
+          args:
+            - --csi-address=/csi/csi.sock
+            - --probe-timeout=3s
+            - --health-port=10300
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+          resources:
+            requests:
+              memory: 20Mi
+              cpu: 5m
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/csi.kubevirt.io/
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory
+        - name: udev
+          hostPath:
+            path: /run/udev
diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/infra_role.yaml b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/infra_role.yaml
@@ -0,0 +1,15 @@
+# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kubevirt-csi
+rules:
+- apiGroups: ["cdi.kubevirt.io"]
+  resources: ["datavolumes"]
+  verbs: ["get", "create", "delete"]
+- apiGroups: ["kubevirt.io"]
+  resources: ["virtualmachineinstances"]
+  verbs: ["list"]
+- apiGroups: ["subresources.kubevirt.io"]
+  resources: ["virtualmachineinstances/addvolume", "virtualmachineinstances/removevolume"]
+  verbs: ["update"]
diff --git a/...l-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/infra_rolebinding.yaml b/...l-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/files/infra_rolebinding.yaml
@@ -0,0 +1,12 @@
+# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubevirt-csi
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubevirt-csi
+subjects:
+- kind: ServiceAccount
+  name: kubevirt-csi
diff --git a/...ator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_controller_clusterrole.yaml b/...ator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_controller_clusterrole.yaml
@@ -0,0 +1,55 @@
+# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubevirt-csi-controller-cr
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["create", "delete", "get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csidrivers"]
+    verbs: ["get", "list", "watch", "update", "create", "patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["security.openshift.io"]
+    resources: ["securitycontextconstraints"]
+    verbs: ["use"]
+    resourceNames: ["privileged"]
diff --git a/...ntrollers/hostedcontrolplane/csi/kubevirt/files/tenant_controller_clusterrolebinding.yaml b/...ntrollers/hostedcontrolplane/csi/kubevirt/files/tenant_controller_clusterrolebinding.yaml
@@ -0,0 +1,12 @@
+# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kubevirt-csi-controller-binding
+subjects:
+  - kind: ServiceAccount
+    name: kubevirt-csi-controller-sa
+roleRef:
+  kind: ClusterRole
+  name: kubevirt-csi-controller-cr
+  apiGroup: rbac.authorization.k8s.io
diff --git a/...e-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_node_clusterrole.yaml b/...e-operator/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_node_clusterrole.yaml
@@ -0,0 +1,37 @@
+# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubevirt-csi-node-cr
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "create", "delete", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csinodeinfos"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["security.openshift.io"]
+    resources: ["securitycontextconstraints"]
+    verbs: ["use"]
+    resourceNames: ["privileged"]
diff --git a/...tor/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_node_clusterrolebinding.yaml b/...tor/controllers/hostedcontrolplane/csi/kubevirt/files/tenant_node_clusterrolebinding.yaml
@@ -0,0 +1,12 @@
+# sourced from https://github.com/kubevirt/csi-driver/tree/main/deploy/split-infra-tenant
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kubevirt-csi-node-binding
+subjects:
+  - kind: ServiceAccount
+    name: kubevirt-csi-node-sa
+roleRef:
+  kind: ClusterRole
+  name: kubevirt-csi-node-cr
+  apiGroup: rbac.authorization.k8s.io