Merge pull request volcano-sh#5 from volcano-sh/codes/support_helm_chart

cmd/controllers/app/server.go

@@ -60,10 +60,10 @@ func Run(opt *options.ServerOption) error {
 		return err
 	}
 
-	JobController := job.NewJobController(config)
+	jobController := job.NewJobController(config)
 
 	run := func(ctx context.Context) {
-		JobController.Run(ctx.Done())
+		jobController.Run(ctx.Done())
 		<-ctx.Done()
 	}
 

hack/e2e-admission-config.yaml

@@ -0,0 +1,44 @@
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validate-volcano-job
+webhooks:
+  - clientConfig:
+      caBundle: {{CA_BUNDLE}}
+
+      # the url should agree with webhook service
+      url: https://{{host}}:{{hostPort}}/jobs
+    failurePolicy: Ignore
+    name: validatejob.volcano.sh
+    rules:
+      - apiGroups:
+          - "batch.volcano.sh"
+        apiVersions:
+          - "v1alpha1"
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - jobs
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: mutate-volcano-job
+webhooks:
+  - clientConfig:
+      caBundle: {{CA_BUNDLE}}
+
+      # the url should agree with webhook service
+      url: https://{{host}}:{{hostPort}}/mutating-jobs
+    failurePolicy: Ignore
+    name: mutatejob.volcano.sh
+    rules:
+      - apiGroups:
+          - "batch.volcano.sh"
+        apiVersions:
+          - "v1alpha1"
+        operations:
+          - CREATE
+        resources:
+          - jobs

hack/e2e-kind-config.yaml

@@ -0,0 +1,27 @@
+# this config file contains all config fields with comments
+kind: Config
+apiVersion: kind.sigs.k8s.io/v1alpha2
+# 1 control plane node and 3 workers
+nodes:
+  # the control plane node config
+  - role: control-plane
+    # patch the generated kubeadm config with some extra settings
+    kubeadmConfigPatches:
+      - |
+        apiVersion: kubeadm.k8s.io/v1beta1
+        kind: ClusterConfiguration
+        networking:
+          serviceSubnet: 10.0.0.0/16
+    # patch it further using a JSON 6902 patch
+    kubeadmConfigPatchesJson6902:
+      - group: kubeadm.k8s.io
+        version: v1beta1
+        kind: ClusterConfiguration
+        patch: |
+          - op: add
+            path: /apiServer/certSANs/-
+            value: my-hostname
+  # the three workers
+  - role: worker
+    # replicas specifes the number of nodes to create with this configuration
+    replicas: 3

hack/run-e2e-kind.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+export VK_ROOT=$(dirname "${BASH_SOURCE}")/..
+export VK_BIN=${VK_ROOT}/_output/bin
+export LOG_LEVEL=3
+export SHOW_VOLCANO_LOGS=${SHOW_VOLCANO_LOGS:-1}
+
+if [ "${CLUSTER_NAME}xxx" != "xxx" ];then
+  export CLUSTER_CONTEXT="--name ${CLUSTER_NAME}"
+fi
+
+export KIND_OPT=${KIND_OPT:="--image kindest/node:v1.13.2-huawei --config ${VK_ROOT}/hack/e2e-kind-config.yaml"}
+
+export KIND_IMAGE=$(echo ${KIND_OPT} |grep -E -o "image \w+\/[^ ]*" | sed "s/image //")
+
+# check if kind installed
+function check-prerequisites {
+  echo "checking prerequisites"
+  which kind >/dev/null 2>&1
+  if [ $? -ne 0 ]; then
+    echo "kind not installed, exiting."
+    exit 1
+  else
+    echo -n "found kind, version: " && kind version
+  fi
+
+  which kubectl >/dev/null 2>&1
+  if [ $? -ne 0 ]; then
+    echo "kubectl not installed, exiting."
+    exit 1
+  else
+    echo -n "found kubectl, " && kubectl version --short --client
+  fi
+}
+
+# check if the images that kind use exists.
+function check-kind-image {
+  docker images | awk '{print $1":"$2}' | grep -q "${KIND_IMAGE}"
+  if [ $? -ne 0 ]; then
+    echo "image: ${KIND_IMAGE} not found."
+    exit 1
+  fi
+}
+
+# spin up cluster with kind command
+function kind-up-cluster {
+  check-prerequisites
+  check-kind-image
+  echo "Running kind: [kind create cluster ${CLUSTER_CONTEXT} ${KIND_OPT}]"
+  kind create cluster ${CLUSTER_CONTEXT} ${KIND_OPT}
+}
+
+function install-volcano {
+  kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_podgroup.yaml
+  kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_queue.yaml
+  kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml
+  kubectl --kubeconfig ${KUBECONFIG} create -f ${VK_ROOT}/installer/chart/volcano-init/templates/bus_v1alpha1_command.yaml
+
+  # TODO: make vk-controllers and vk-scheduler run in container / in k8s
+  # start controller
+  nohup ${VK_BIN}/vk-controllers --kubeconfig ${KUBECONFIG} --logtostderr --v ${LOG_LEVEL} > controller.log 2>&1 &
+  echo $! > vk-controllers.pid
+
+  # start scheduler
+  nohup ${VK_BIN}/vk-scheduler --kubeconfig ${KUBECONFIG} --scheduler-conf=example/kube-batch-conf.yaml --logtostderr --v ${LOG_LEVEL} > scheduler.log 2>&1 &
+  echo $! > vk-scheduler.pid
+}
+
+function uninstall-volcano {
+  kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_podgroup.yaml
+  kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/scheduling_v1alpha1_queue.yaml
+  kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml
+  kubectl --kubeconfig ${KUBECONFIG} delete -f ${VK_ROOT}/installer/chart/volcano-init/templates/bus_v1alpha1_command.yaml
+
+  kill -9 $(cat vk-controllers.pid)
+  kill -9 $(cat vk-scheduler.pid)
+  rm vk-controllers.pid vk-scheduler.pid
+}
+
+# clean up
+function cleanup {
+  uninstall-volcano
+
+  echo "Running kind: [kind delete cluster ${CLUSTER_CONTEXT}]"
+  kind delete cluster ${CLUSTER_CONTEXT}
+
+  if [ ${SHOW_VOLCANO_LOGS} -eq 1 ]; then
+    echo "===================================================================================="
+    echo "=============================>>>>> Scheduler Logs <<<<<============================="
+    echo "===================================================================================="
+
+    cat scheduler.log
+
+    echo "===================================================================================="
+    echo "=============================>>>>> Controller Logs <<<<<============================"
+    echo "===================================================================================="
+
+    cat controller.log
+  fi
+}
+
+echo $* | grep -E -q "\-\-help|\-h"
+if [ $? -eq 0 ]; then
+  echo "Customize the kind-cluster name:
+
+    export CLUSTER_NAME=<custom cluster name>
+
+Customize kind options other than --name:
+
+    export KIND_OPT=<kind options>
+
+Disable displaying volcano component logs:
+
+    export SHOW_VOLCANO_LOGS=0
+
+If you don't have kindest/node:v1.13.2-huawei on the host, checkout the following url to build.
+
+    http://code-cbu.huawei.com/CBU-PaaS/Community/K8S/kind/tags/v0.1.0-huawei
+"
+  exit 0
+fi
+
+
+trap cleanup EXIT
+
+
+kind-up-cluster
+
+KUBECONFIG="$(kind get kubeconfig-path ${CLUSTER_CONTEXT})"
+
+install-volcano
+
+# Run e2e test
+cd ${VK_ROOT}
+KUBECONFIG=${KUBECONFIG} go test ./test/e2e -v -timeout 30m

hack/run-e2e.sh

@@ -1,48 +1,62 @@
 #!/bin/bash
 
 export PATH="${HOME}/.kubeadm-dind-cluster:${PATH}"
-export VK_BIN=_output/bin
-export LOG_LEVEL=3
+export MASTER="http://127.0.0.1:8080"
+export VK_BIN=$PWD/_output/bin
+export LOG_LEVEL=2
 export NUM_NODES=3
+export CERT_PATH=/etc/kubernetes/pki
+export HOST=localhost
+export HOSTPORT=32222
 
-dind_url=https://cdn.rawgit.com/kubernetes-sigs/kubeadm-dind-cluster/master/fixed/dind-cluster-v1.12.sh
-dind_dest=./hack/dind-cluster-v1.12.sh
+kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/scheduling_v1alpha1_podgroup.yaml
+kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/scheduling_v1alpha1_queue.yaml
+kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml
+kubectl --server=${MASTER} apply -f installer/chart/volcano-init/templates/bus_v1alpha1_command.yaml
 
-# start k8s dind cluster
-curl ${dind_url} --output ${dind_dest}
-chmod +x ${dind_dest}
-${dind_dest} up
+# config admission-controller TODO: make it easier to deploy
+CA_BUNDLE=`kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n'`
+sed -i "s|{{CA_BUNDLE}}|$CA_BUNDLE|g" hack/e2e-admission-config.yaml
+sed -i "s|{{host}}|${HOST}|g" hack/e2e-admission-config.yaml
+sed -i "s|{{hostPort}}|${HOSTPORT}|g" hack/e2e-admission-config.yaml
 
-kubectl create -f config/crds/scheduling_v1alpha1_podgroup.yaml
-kubectl create -f config/crds/scheduling_v1alpha1_queue.yaml
-kubectl create -f config/crds/batch_v1alpha1_job.yaml
-kubectl create -f config/crds/bus_v1alpha1_command.yaml
+kubectl create -f hack/e2e-admission-config.yaml
 
 # start controller
-nohup ${VK_BIN}/vk-controllers --kubeconfig ${HOME}/.kube/config --logtostderr --v ${LOG_LEVEL} > controller.log 2>&1 &
+nohup ${VK_BIN}/vk-controllers --kubeconfig ${HOME}/.kube/config --master=${MASTER} --logtostderr --v ${LOG_LEVEL} > controller.log 2>&1 &
 
 # start scheduler
-nohup ${VK_BIN}/vk-scheduler --kubeconfig ${HOME}/.kube/config --scheduler-conf=example/kube-batch-conf.yaml --logtostderr --v ${LOG_LEVEL} > scheduler.log 2>&1 &
+nohup ${VK_BIN}/vk-scheduler --kubeconfig ${HOME}/.kube/config --scheduler-conf=example/kube-batch-conf.yaml --master=${MASTER} --logtostderr --v ${LOG_LEVEL} > scheduler.log 2>&1 &
+
+# start admission-controller
+nohup ${VK_BIN}/vk-admission --tls-cert-file=${CERT_PATH}/apiserver.crt --tls-private-key-file=${CERT_PATH}/apiserver.key --kubeconfig ${HOME}/.kube/config --port ${HOSTPORT} --logtostderr --v ${LOG_LEVEL} > admission.log 2>&1 &
 
 # clean up
 function cleanup {
-    killall -9 vk-scheduler vk-controllers
-    ./hack/dind-cluster-v1.12.sh down
-
-    echo "===================================================================================="
-    echo "=============================>>>>> Scheduler Logs <<<<<============================="
-    echo "===================================================================================="
-
-    cat scheduler.log
+    killall -9 -r vk-scheduler -r vk-controllers -r vk-admission
+
+    if [[ -f scheduler.log ]] ; then
+        echo "===================================================================================="
+        echo "=============================>>>>> Scheduler Logs <<<<<============================="
+        echo "===================================================================================="
+        cat scheduler.log
+    fi
+
+    if [[ -f controller.log ]] ; then
+        echo "===================================================================================="
+        echo "=============================>>>>> Controller Logs <<<<<============================"
+        echo "===================================================================================="
+        cat controller.log
+    fi
 
     echo "===================================================================================="
-    echo "=============================>>>>> Controller Logs <<<<<============================"
+    echo "=============================>>>>> admission Logs <<<<<============================"
     echo "===================================================================================="
 
-    cat controller.log
+    cat admission.log
 }
 
 trap cleanup EXIT
 
 # Run e2e test
-go test ./test/e2e -v
+go test ./test/e2e -v -timeout 30m

installer/chart/volcano-init/Chart.yaml

@@ -0,0 +1,4 @@
+name: volcano-init
+version: 0.0.1
+description: volcano crds and admission-controller config
+apiVersion: v1

config/crds/batch_v1alpha1_job.yaml → installer/chart/volcano-init/templates/batch_v1alpha1_job.yaml

@@ -76,9 +76,9 @@ spec:
                 type: object
               type: array
             schedulerName:
-              description: SchedulerName is the default value of `taskSpecs.template.spec.schedulerName`.
+              description: SchedulerName is the default value of `tasks.template.spec.schedulerName`.
               type: string
-            taskSpecs:
+            tasks:
               description: Tasks specifies the task specification of Job
               items:
                 properties:
@@ -139,6 +139,10 @@ spec:
               description: The number of running pods.
               format: int32
               type: integer
+            version:
+              description: Job's current version
+              format: int32
+              type: integer
             state:
               description: Current state of Job.
               properties:
@@ -156,6 +160,8 @@ spec:
               type: object
           type: object
   version: v1alpha1
+  subresources:
+    status: {}
 status:
   acceptedNames:
     kind: ""

installer/chart/volcano-init/values.yaml

@@ -0,0 +1,2 @@
+basic:
+  namespace: default

installer/chart/volcano/Chart.yaml

@@ -0,0 +1,4 @@
+name: volcano
+version: 0.0.1
+description: volcano
+apiVersion: v1