Add other logic files

Co-authored-by: Yingshan Liu <[email protected]>
Signed-off-by: ichbinblau <[email protected]>

Makefile

@@ -116,16 +116,20 @@ push-release-images: release-image.amd64 release-image.arm64v8
 update-vendor:
 	hack/update-vendor.sh
 
+.PHONY: build-diskioplugin
+build-diskioplugin:
+	$(COMMONENVVAR) CGO_ENABLED=1 go build -buildmode=plugin -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o pkg/diskioaware/sampleplugin/foo/foo.so ./pkg/diskioaware/sampleplugin/foo/foo.go
+
 .PHONY: unit-test
-unit-test: install-envtest
+unit-test: install-envtest build-diskioplugin
 	hack/unit-test.sh $(ARGS)
 
 .PHONY: install-envtest
 install-envtest:
 	hack/install-envtest.sh
 
 .PHONY: integration-test
-integration-test: install-envtest
+integration-test: install-envtest build-diskioplugin
 	$(INTEGTESTENVVAR) hack/integration-test.sh $(ARGS)
 
 .PHONY: verify
@@ -139,3 +143,4 @@ verify:
 .PHONY: clean
 clean:
 	rm -rf ./bin
+	rm -f ./pkg/diskioaware/sampleplugin/foo/foo.so

README.md

@@ -28,6 +28,7 @@ The kube-scheduler binary includes the below list of plugins. They can be config
 
 * [Capacity Scheduling](pkg/capacityscheduling/README.md)
 * [Coscheduling](pkg/coscheduling/README.md)
+* [Disk IO aware Scheduling](pkg/diskioaware/README.md)
 * [Node Resources](pkg/noderesources/README.md)
 * [Node Resource Topology](pkg/noderesourcetopology/README.md)
 * [Preemption Toleration](pkg/preemptiontoleration/README.md)

cmd/scheduler/main.go

@@ -26,6 +26,7 @@ import (
 
 	"sigs.k8s.io/scheduler-plugins/pkg/capacityscheduling"
 	"sigs.k8s.io/scheduler-plugins/pkg/coscheduling"
+	"sigs.k8s.io/scheduler-plugins/pkg/diskioaware"
 	"sigs.k8s.io/scheduler-plugins/pkg/networkaware/networkoverhead"
 	"sigs.k8s.io/scheduler-plugins/pkg/networkaware/topologicalsort"
 	"sigs.k8s.io/scheduler-plugins/pkg/noderesources"
@@ -58,6 +59,7 @@ func main() {
 		app.WithPlugin(targetloadpacking.Name, targetloadpacking.New),
 		app.WithPlugin(lowriskovercommitment.Name, lowriskovercommitment.New),
 		app.WithPlugin(sysched.Name, sysched.New),
+		app.WithPlugin(diskioaware.Name, diskioaware.New),
 		// Sample plugins below.
 		// app.WithPlugin(crossnodepreemption.Name, crossnodepreemption.New),
 		app.WithPlugin(podstate.Name, podstate.New),

hack/verify-crdgen.sh

@@ -30,7 +30,7 @@ CONTROLLER_GEN="${GOPATH}/bin/controller-gen"
 go install sigs.k8s.io/controller-tools/cmd/[email protected]
 
 # Generate CRD
-api_paths="./apis/scheduling/v1alpha1/...;./vendor/github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/...;./vendor/github.com/diktyo-io/appgroup-api/pkg/apis/...;./vendor/github.com/diktyo-io/networktopology-api/pkg/apis/...;./vendor/sigs.k8s.io/security-profiles-operator/api/seccompprofile/v1beta1/..."
+api_paths="./apis/scheduling/v1alpha1/...;./vendor/github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/...;./vendor/github.com/diktyo-io/appgroup-api/pkg/apis/...;./vendor/github.com/diktyo-io/networktopology-api/pkg/apis/...;./vendor/sigs.k8s.io/security-profiles-operator/api/seccompprofile/v1beta1/...;./vendor/github.com/intel/cloud-resource-scheduling-and-isolation/pkg/api/diskio/v1alpha1/..."
 
 ${CONTROLLER_GEN} ${CRD_OPTIONS} paths="${api_paths}" output:dir="./manifests/crds"
 

manifests/crds/diskio.x-k8s.io_nodediskdevices.yaml

@@ -0,0 +1,93 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.14.0
+  name: nodediskdevices.diskio.x-k8s.io
+spec:
+  group: diskio.x-k8s.io
+  names:
+    kind: NodeDiskDevice
+    listKind: NodeDiskDeviceList
+    plural: nodediskdevices
+    singular: nodediskdevice
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              devices:
+                additionalProperties:
+                  properties:
+                    capacity:
+                      description: Profile result of io bandwidth capacity
+                      properties:
+                        read:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          description: Normalized read IO throughput capacity
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        total:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          description: Normalized total IO throughput capacity
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        write:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          description: Normalized write IO throughput capacity
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                      type: object
+                    model:
+                      description: Device model
+                      type: string
+                    name:
+                      description: Device name
+                      type: string
+                    type:
+                      description: Default or not
+                      type: string
+                    vendor:
+                      description: Device vendor
+                      type: string
+                  required:
+                  - name
+                  type: object
+                type: object
+              nodeName:
+                type: string
+            type: object
+          status:
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

manifests/crds/diskio.x-k8s.io_nodediskiostats.yaml

@@ -0,0 +1,91 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.14.0
+  name: nodediskiostats.diskio.x-k8s.io
+spec:
+  group: diskio.x-k8s.io
+  names:
+    kind: NodeDiskIOStats
+    listKind: NodeDiskIOStatsList
+    plural: nodediskiostats
+    singular: nodediskiostats
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              nodeName:
+                type: string
+              reservedPods:
+                description: a slice of reserved pod uids
+                items:
+                  type: string
+                type: array
+            required:
+            - nodeName
+            type: object
+          status:
+            description: NodeDiskIOStatsStatus defines the observed state of node
+              disks
+            properties:
+              allocableBandwidth:
+                additionalProperties:
+                  properties:
+                    read:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      description: Normalized read IO throughput capacity
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    total:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      description: Normalized total IO throughput capacity
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                    write:
+                      anyOf:
+                      - type: integer
+                      - type: string
+                      description: Normalized write IO throughput capacity
+                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                      x-kubernetes-int-or-string: true
+                  type: object
+                description: the key of the map is the device id
+                type: object
+              observedGeneration:
+                description: ObservedGeneration is the most recent generation observed
+                  by this IO Driver.
+                format: int64
+                type: integer
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

manifests/diskioaware/cluster-role.yaml

@@ -0,0 +1,73 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: diskio-aware-scheduler
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: diskioaware-handler
+rules:
+- apiGroups: ["diskio.x-k8s.io"]
+  resources: ["nodediskdevices"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["diskio.x-k8s.io"]
+  resources: 
+  - nodediskiostats
+  - nodediskiostats/status
+  verbs: ["create","get", "watch", "list", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: diskio-aware-scheduler-as-kube-scheduler
+subjects:
+  - kind: ServiceAccount
+    name: diskio-aware-scheduler
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: diskioaware-handler
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: my-scheduler-as-volume-scheduler
+subjects:
+- kind: ServiceAccount
+  name: diskio-aware-scheduler
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: system:volume-scheduler
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: diskio-aware-scheduler-as-kube-scheduler
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: diskio-aware-scheduler
+    namespace: kube-system
+roleRef:
+  kind: Role
+  name: extension-apiserver-authentication-reader
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: diskioaware
+  namespace: kube-system
+subjects:
+- kind: ServiceAccount
+  name: diskio-aware-scheduler
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: system:kube-scheduler
+  apiGroup: rbac.authorization.k8s.io

manifests/diskioaware/deploy.yaml

@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    component: scheduler
+    tier: control-plane
+  name: diskio-aware-scheduler
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      component: scheduler
+      tier: control-plane
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        component: scheduler
+        tier: control-plane
+        version: second
+    spec:
+      serviceAccountName: diskio-aware-scheduler
+      containers:
+        - image: localhost:5000/scheduler-plugins/kube-scheduler:latest
+          imagePullPolicy: IfNotPresent
+          command:
+          - /bin/kube-scheduler
+          - --bind-address=127.0.0.1
+          - --config=/etc/kubernetes/scheduler-config/scheduler-config.yaml
+          name: diskio-aware-scheduler
+          securityContext:
+            privileged: false
+          volumeMounts:
+          - mountPath: /etc/kubernetes/scheduler-config
+            name: diskio-aware-scheduler-config-vol
+          - mountPath: /etc/kubernetes/disk-config
+            name: diskio-aware-scheduler-diskmodel-vol
+      hostNetwork: false
+      hostPID: false
+      volumes:
+      - configMap:
+          name: diskio-aware-scheduler-config
+        name: diskio-aware-scheduler-config-vol
+      - configMap:
+          name: normalization-func
+        name: diskio-aware-scheduler-diskmodel-vol

manifests/diskioaware/diskVendors

@@ -0,0 +1 @@
+[{"vendor":"Intel","model":"P4510","url":"http://10.244.2.1:8080/foo/foo.so"}]

manifests/diskioaware/diskmodel-configmap.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: "ioi-system"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: normalization-func
+  namespace: ioi-system
+data:
+  diskVendors: '[{"vendor":"Intel","model":"P4510","url":"http://localhost:8080/foo/foo.so"}]'