diff --git a/Makefile b/Makefile index 27f7310b..16ca2181 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ SCRIPTS_DIR := $(ROOT_DIR)/scripts KUDO_TOOLS_DIR := $(ROOT_DIR)/shared SPARK_OPERATOR_DIR := $(ROOT_DIR)/spark-on-k8s-operator -export KONVOY_VERSION ?= v1.5.0 +export KONVOY_VERSION ?= v1.6.0-rc.2 export CLUSTER_CONFIG_YAML ?= $(ROOT_DIR)/cluster.template.yaml export WORKER_NODE_INSTANCE_TYPE ?= m5.2xlarge export WORKER_NODE_COUNT ?= 5 @@ -171,7 +171,7 @@ clean-docker: clean-all: clean-docker clean-all: rm -f *.pem *.pub *-created aws_credentials - rm -rf state runs .konvoy-* *checksum cluster.*yaml* inventory.yaml admin.conf + rm -rf state runs .konvoy-* *checksum cluster.yaml* inventory.yaml admin.conf # function for extracting the value of an AWS property passed as an argument define get_aws_credential diff --git a/cluster.template.yaml b/cluster.template.yaml index 8d59a03e..b6fc94ec 100644 --- a/cluster.template.yaml +++ b/cluster.template.yaml @@ -52,7 +52,7 @@ spec: user: centos publicKeyFile: kudo-spark-ssh.pub privateKeyFile: kudo-spark-ssh.pem - version: v1.5.0 + version: v1.6.0-rc.2 --- kind: ClusterConfiguration apiVersion: konvoy.mesosphere.io/v1beta2 @@ -61,7 +61,7 @@ metadata: creationTimestamp: "2020-01-28T23:15:38Z" spec: kubernetes: - version: 1.17.8 + version: 1.18.9 controlPlane: controlPlaneEndpointOverride: "" certificate: {} @@ -80,7 +80,7 @@ spec: - NodeRestriction containerNetworking: calico: - version: v3.13.4 + version: v3.16.0 encapsulation: ipip mtu: 1480 containerRuntime: @@ -92,7 +92,7 @@ spec: - name: worker addons: - configRepository: https://github.com/mesosphere/kubernetes-base-addons - configVersion: stable-1.17-2.0.2 + configVersion: testing-2.5.0-3 addonsList: - name: awsebscsiprovisioner enabled: true @@ -181,4 +181,4 @@ spec: addonsList: - name: kommander enabled: false - version: v1.5.0 + version: v1.6.0-rc.2 diff --git a/operators b/operators index 006fe389..48124c1e 160000 --- a/operators +++ b/operators @@ -1 +1 @@ -Subproject commit 006fe3898caebb198923ea2694af5872d71e0040 +Subproject commit 48124c1e8bc92dff7b9885681142e0361213685b diff --git a/tests/test-app-scheduling-with-volcano/00-install-volcano.yaml b/tests/test-app-scheduling-with-volcano/00-install-volcano.yaml index c2e0415f..fd355d37 100644 --- a/tests/test-app-scheduling-with-volcano/00-install-volcano.yaml +++ b/tests/test-app-scheduling-with-volcano/00-install-volcano.yaml @@ -36,57 +36,54 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: volcano-scheduler rules: -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "get", "list", "watch", "delete"] -- apiGroups: ["batch.volcano.sh"] - resources: ["jobs"] - verbs: ["get", "list", "watch", "update", "delete"] -- apiGroups: ["batch.volcano.sh"] - resources: ["jobs/status"] - verbs: ["update", "patch"] -- apiGroups: [""] - resources: ["events"] - verbs: ["create", "list", "watch", "update", "patch"] -- apiGroups: [""] - resources: ["pods", "pods/status"] - verbs: ["create", "get", "list", "watch", "update", "bind", "updateStatus", "delete"] -- apiGroups: [""] - resources: ["pods/binding"] - verbs: ["create"] -- apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["list", "watch"] -- apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["list", "watch"] -- apiGroups: ["scheduling.incubator.k8s.io", "scheduling.sigs.dev"] - resources: ["podgroups"] - verbs: ["list", "watch", "update"] -- apiGroups: [""] - resources: ["namespaces"] - verbs: ["list", "watch"] -- apiGroups: [""] - resources: ["resourcequotas"] - verbs: ["list", "watch"] -- apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["list", "watch"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["list", "watch"] -- apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] - verbs: ["list", "watch"] -- apiGroups: ["scheduling.k8s.io"] - resources: ["priorityclasses"] - verbs: ["get", "list", "watch"] -- apiGroups: ["scheduling.incubator.k8s.io", "scheduling.sigs.dev"] - resources: ["queues"] - verbs: ["get", "list", "watch", "create", "delete"] -- apiGroups: ["scheduling.incubator.k8s.io", "scheduling.sigs.dev"] - resources: ["podgroups"] - verbs: ["list", "watch", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "update", "delete"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["pods", "pods/status"] + verbs: ["create", "get", "list", "watch", "update", "patch", "bind", "updateStatus", "delete"] + - apiGroups: [""] + resources: ["pods/binding"] + verbs: ["create"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["list", "watch"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["list", "watch"] + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "watch"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["list", "watch"] + - apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] + resources: ["queues"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] + resources: ["podgroups"] + verbs: ["list", "watch", "update"] --- kind: ClusterRoleBinding @@ -94,9 +91,9 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: volcano-scheduler-role subjects: -- kind: ServiceAccount - name: volcano-scheduler - namespace: volcano-system + - kind: ServiceAccount + name: volcano-scheduler + namespace: volcano-system roleRef: kind: ClusterRole name: volcano-scheduler @@ -121,23 +118,22 @@ spec: app: volcano-scheduler spec: serviceAccount: volcano-scheduler - containers: - - name: volcano-scheduler - image: volcanosh/vc-scheduler:v0.2 - args: - - --alsologtostderr - - --scheduler-conf=/volcano.scheduler/volcano-scheduler.conf - - -v=3 - - 2>&1 - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: scheduler-config - mountPath: /volcano.scheduler + - name: volcano-scheduler + image: volcanosh/vc-scheduler:latest + args: + - --logtostderr + - --scheduler-conf=/volcano.scheduler/volcano-scheduler.conf + - -v=3 + - 2>&1 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: scheduler-config + mountPath: /volcano.scheduler volumes: - - name: scheduler-config - configMap: - name: volcano-scheduler-configmap + - name: scheduler-config + configMap: + name: volcano-scheduler-configmap --- # Source: volcano/templates/admission.yaml @@ -152,31 +148,31 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: volcano-admission rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch"] -- apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] - verbs: ["get", "list", "watch", "create", "update"] -# Rules below is used generate admission service secret -- apiGroups: ["certificates.k8s.io"] - resources: ["certificatesigningrequests"] - verbs: ["get", "list", "create", "delete"] -- apiGroups: ["certificates.k8s.io"] - resources: ["certificatesigningrequests/approval"] - verbs: ["create", "update"] -- apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "get", "patch"] -- apiGroups: ["scheduling.incubator.k8s.io", "scheduling.sigs.dev"] - resources: ["queues"] - verbs: ["get", "list"] -- apiGroups: [""] - resources: ["services"] - verbs: ["get"] -- apiGroups: ["scheduling.incubator.k8s.io", "scheduling.sigs.dev"] - resources: ["podgroups"] - verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["get", "list", "watch", "create", "update"] + # Rules below is used generate admission service secret + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests/approval"] + verbs: ["create", "update"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "patch"] + - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] + resources: ["queues"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get"] + - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] + resources: ["podgroups"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding @@ -184,9 +180,9 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: volcano-admission-role subjects: -- kind: ServiceAccount - name: volcano-admission - namespace: volcano-system + - kind: ServiceAccount + name: volcano-admission + namespace: volcano-system roleRef: kind: ClusterRole name: volcano-admission @@ -211,30 +207,29 @@ spec: app: volcano-admission spec: serviceAccount: volcano-admission - containers: - - args: - - --tls-cert-file=/admission.local.config/certificates/tls.crt - - --tls-private-key-file=/admission.local.config/certificates/tls.key - - --ca-cert-file=/admission.local.config/certificates/ca.crt - - --webhook-namespace=volcano-system - - --webhook-service-name=volcano-admission-service - - --alsologtostderr - - --port=443 - - -v=4 - - 2>&1 - image: volcanosh/vc-admission:v0.2 - imagePullPolicy: IfNotPresent - name: admission - volumeMounts: - - mountPath: /admission.local.config/certificates - name: admission-certs - readOnly: true + - args: + - --tls-cert-file=/admission.local.config/certificates/tls.crt + - --tls-private-key-file=/admission.local.config/certificates/tls.key + - --ca-cert-file=/admission.local.config/certificates/ca.crt + - --webhook-namespace=volcano-system + - --webhook-service-name=volcano-admission-service + - --logtostderr + - --port=8443 + - -v=4 + - 2>&1 + image: volcanosh/vc-webhook-manager:latest + imagePullPolicy: IfNotPresent + name: admission + volumeMounts: + - mountPath: /admission.local.config/certificates + name: admission-certs + readOnly: true volumes: - - name: admission-certs - secret: - defaultMode: 420 - secretName: volcano-admission-secret + - name: admission-certs + secret: + defaultMode: 420 + secretName: volcano-admission-secret --- apiVersion: v1 @@ -246,9 +241,9 @@ metadata: namespace: volcano-system spec: ports: - - port: 443 - protocol: TCP - targetPort: 443 + - port: 443 + protocol: TCP + targetPort: 8443 selector: app: volcano-admission sessionAffinity: None @@ -268,11 +263,11 @@ spec: serviceAccountName: volcano-admission restartPolicy: Never containers: - - name: main - image: volcanosh/vc-admission:v0.2 - imagePullPolicy: IfNotPresent - command: ["./gen-admission-secret.sh", "--service", "volcano-admission-service", "--namespace", - "volcano-system", "--secret", "volcano-admission-secret"] + - name: main + image: volcanosh/vc-webhook-manager:latest + imagePullPolicy: IfNotPresent + command: ["./gen-admission-secret.sh", "--service", "volcano-admission-service", "--namespace", + "volcano-system", "--secret", "volcano-admission-secret"] --- # Source: volcano/templates/controllers.yaml @@ -288,42 +283,45 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: volcano-controllers rules: -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "get", "list", "watch", "delete"] -- apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["create", "get", "list", "watch", "delete", "update"] -- apiGroups: ["batch.volcano.sh"] - resources: ["jobs"] - verbs: ["get", "list", "watch", "update", "delete"] -- apiGroups: ["batch.volcano.sh"] - resources: ["jobs/status"] - verbs: ["update", "patch"] -- apiGroups: ["bus.volcano.sh"] - resources: ["commands"] - verbs: ["get", "list", "watch", "delete"] -- apiGroups: [""] - resources: ["events"] - verbs: ["create", "list", "watch", "update", "patch"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["create", "get", "list", "watch", "update", "bind", "delete"] -- apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "create"] -- apiGroups: [""] - resources: ["services"] - verbs: ["get", "list", "watch", "create", "delete"] -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "delete", "update"] -- apiGroups: ["scheduling.incubator.k8s.io", "scheduling.sigs.dev"] - resources: ["podgroups", "queues", "queues/status"] - verbs: ["get", "list", "watch", "create", "delete", "update"] -- apiGroups: ["scheduling.k8s.io"] - resources: ["priorityclasses"] - verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "update", "delete"] + - apiGroups: ["batch.volcano.sh"] + resources: ["jobs/status", "jobs/finalizers"] + verbs: ["update", "patch"] + - apiGroups: ["bus.volcano.sh"] + resources: ["commands"] + verbs: ["get", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "update", "bind", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] + resources: ["podgroups", "queues", "queues/status"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "create"] --- kind: ClusterRoleBinding @@ -331,9 +329,9 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: volcano-controllers-role subjects: -- kind: ServiceAccount - name: volcano-controllers - namespace: volcano-system + - kind: ServiceAccount + name: volcano-controllers + namespace: volcano-system roleRef: kind: ClusterRole name: volcano-controllers @@ -358,15 +356,14 @@ spec: app: volcano-controller spec: serviceAccount: volcano-controllers - containers: - - name: volcano-controllers - image: volcanosh/vc-controllers:v0.2 - args: - - --alsologtostderr - - -v=4 - - 2>&1 - imagePullPolicy: "IfNotPresent" + - name: volcano-controllers + image: volcanosh/vc-controller-manager:latest + args: + - --logtostderr + - -v=4 + - 2>&1 + imagePullPolicy: "IfNotPresent" --- # Source: volcano/templates/batch_v1alpha1_job.yaml @@ -374,19 +371,18 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: jobs.batch.volcano.sh - annotations: - "helm.sh/hook": crd-install spec: group: batch.volcano.sh names: kind: Job plural: jobs shortNames: - - vcjob - - vj + - vcjob + - vj scope: Namespaced validation: openAPIV3Schema: + type: object properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -417,9 +413,10 @@ spec: type: string volumeClaimName: description: The name of the volume claim. + type: string type: object required: - - mountPath + - mountPath type: array minAvailable: description: The minimal available pods to run for this Job @@ -441,6 +438,8 @@ spec: description: The Events recorded by scheduler; the controller takes actions according to this Events. type: array + items: + type: string timeout: description: Timeout is the grace period for controller to take actions. Default to nil (take action immediately). @@ -453,8 +452,6 @@ spec: plugins: description: Enabled task plugins when creating job. type: object - additionalProperties: - type: array tasks: description: Tasks specifies the task specification of Job items: @@ -479,6 +476,8 @@ spec: description: The Events recorded by scheduler; the controller takes actions according to this Events. type: array + items: + type: string timeout: description: Timeout is the grace period for controller to take actions. Default to nil (take action immediately). @@ -507,7 +506,7 @@ spec: status: description: Current status of Job properties: - Succeeded: + succeeded: description: The number of pods which reached phase Succeeded. format: int32 type: integer @@ -535,7 +534,7 @@ spec: description: The number that volcano retried to submit the job. format: int32 type: integer - ControlledResources: + controlledResources: description: All of the resources that are controlled by this job. type: object additionalProperties: @@ -554,6 +553,10 @@ spec: description: Unique, one-word, CamelCase reason for the condition's last transition. type: string + lastTransitionTime: + description: The time of last state transition. + format: date-time + type: string type: object type: object version: v1alpha1 @@ -572,8 +575,6 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: commands.bus.volcano.sh - annotations: - "helm.sh/hook": crd-install spec: group: bus.volcano.sh names: @@ -582,6 +583,7 @@ spec: scope: Namespaced validation: openAPIV3Schema: + type: object properties: action: description: Action defines the action that will be took to the target object. @@ -616,103 +618,19 @@ status: storedVersions: [] --- -# Source: volcano/templates/scheduling_v1alpha1_podgroup.yaml +# Source: volcano/templates/scheduling_v1beta1_podgroup.yaml apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: podgroups.scheduling.incubator.k8s.io - annotations: - "helm.sh/hook": crd-install + name: podgroups.scheduling.volcano.sh spec: - group: scheduling.incubator.k8s.io - names: - kind: PodGroup - plural: podgroups - scope: Namespaced - validation: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - minMember: - format: int32 - type: integer - type: object - status: - properties: - succeeded: - format: int32 - type: integer - failed: - format: int32 - type: integer - running: - format: int32 - type: integer - type: object - type: object - version: v1alpha1 - ---- -# Source: volcano/templates/scheduling_v1alpha1_queue.yaml -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: queues.scheduling.incubator.k8s.io - annotations: - "helm.sh/hook": crd-install -spec: - group: scheduling.incubator.k8s.io - names: - kind: Queue - plural: queues - scope: Cluster - validation: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - weight: - format: int32 - type: integer - type: object - type: object - version: v1alpha1 - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -# Source: volcano/templates/scheduling_v1alpha2_podgroup.yaml -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: podgroups.scheduling.sigs.dev -spec: - group: scheduling.sigs.dev + group: scheduling.volcano.sh names: kind: PodGroup plural: podgroups shortNames: - - pg - - podgroup-v1alpha2 + - pg + - podgroup-v1beta1 scope: Namespaced validation: openAPIV3Schema: @@ -746,22 +664,22 @@ spec: type: integer type: object type: object - version: v1alpha2 + version: v1beta1 --- -# Source: volcano/templates/scheduling_v1alpha2_queue.yaml +# Source: volcano/templates/scheduling_v1beta1_queue.yaml apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: queues.scheduling.sigs.dev + name: queues.scheduling.volcano.sh spec: - group: scheduling.sigs.dev + group: scheduling.volcano.sh names: kind: Queue plural: queues shortNames: - - q - - queue-v1alpha2 + - q + - queue-v1beta1 scope: Cluster validation: openAPIV3Schema: @@ -777,9 +695,13 @@ spec: weight: format: int32 type: integer + capability: + type: object type: object status: properties: + state: + type: string unknown: format: int32 type: integer @@ -789,8 +711,11 @@ spec: running: format: int32 type: integer + inqueue: + format: int32 + type: integer type: object type: object - version: v1alpha2 + version: v1beta1 subresources: status: {}