add volume controller and volume replica node level taint toleration

GNUmakefile

@@ -17,8 +17,9 @@ EXTERNAL_TOOLS=\
 # list only our .go files i.e. exlcudes any .go files from the vendor directory
 GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
 
-# Specify the name for the maya binary
+# Specify the name for the binaries
 MAYACTL=maya
+APISERVER=apiserver
 
 # Specify the date o build
 BUILD_DATE = $(shell date +'%Y%m%d%H%M%S')
@@ -29,6 +30,9 @@ dev: format
 	@MAYACTL=${MAYACTL} MAYA_DEV=1 sh -c "'$(PWD)/buildscripts/build.sh'"
 
 bin:
+	@echo "----------------------------"
+	@echo "--> maya                    "
+	@echo "----------------------------"
 	@MAYACTL=${MAYACTL} sh -c "'$(PWD)/buildscripts/build.sh'"
 
 initialize: bootstrap
@@ -39,6 +43,7 @@ deps:
 clean:
 	rm -rf bin
 	rm -rf ${GOPATH}/bin/${MAYACTL}
+	rm -rf ${GOPATH}/bin/${APISERVER}
 	rm -rf ${GOPATH}/pkg/*
 
 release:
@@ -97,6 +102,20 @@ maya-agent:
 
 # Use this to build only the maya apiserver. 
 apiserver:
-	GOOS=linux go build ./cmd/apiserver
-
-.PHONY: all apiserver bin cov integ test vet maya-agent test-nodep
+	@echo "----------------------------"
+	@echo "--> apiserver               "
+	@echo "----------------------------"
+	@CTLNAME=${APISERVER} sh -c "'$(PWD)/buildscripts/apiserver/build.sh'"
+
+# Currently both mayactl & apiserver binaries are pushed into
+# m-apiserver image. This is going to be decoupled soon.
+apiserver-image: bin apiserver
+	@echo "----------------------------"
+	@echo "--> apiserver image         "
+	@echo "----------------------------"
+	@cp bin/apiserver/${APISERVER} buildscripts/apiserver/
+	@cp bin/${MAYACTL} buildscripts/apiserver/
+	@cd buildscripts/apiserver && sudo docker build -t openebs/m-apiserver:ci --build-arg BUILD_DATE=${BUILD_DATE} .
+	@sh buildscripts/apiserver/push
+
+.PHONY: all bin cov integ test vet maya-agent test-nodep apiserver apiserver-image

Vagrantfile

@@ -9,7 +9,7 @@ Vagrant.require_version ">= 1.6.0"
 M_NODES = ENV['M_NODES'] || 1
 
 # Maya Memory & CPUs
-M_MEM = ENV['M_MEM'] || 1024
+M_MEM = ENV['M_MEM'] || 2048
 M_CPUS = ENV['M_CPUS'] || 1
 
 # Generic installer script common for server(s) & client(s)
@@ -75,7 +75,10 @@ def configureVM(vmCfg, hostname, cpus, mem)
     vb.cpus = cpus
     vb.customize ["modifyvm", :id, "--cableconnected1", "on"]
   end
-
+
+  # ensure docker is installed
+  vmCfg.vm.provision "docker"
+
   # sync your laptop's development with this Vagrant VM
   vmCfg.vm.synced_folder '.', '/opt/gopath/src/github.com/openebs/maya'
 

buildscripts/apiserver/Dockerfile

@@ -0,0 +1,30 @@
+#
+# This Dockerfile builds a recent maya api server using the latest binary from
+# maya api server's releases.
+#
+
+FROM ubuntu:16.04
+
+# TODO: The following env variables should be auto detected. 
+ENV MAYA_API_SERVER_NETWORK="eth0"
+
+RUN apt-get update && apt-get install -y \
+    iproute2 \
+    curl \
+    net-tools \
+    watch \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /etc/apiserver/orchprovider
+RUN mkdir -p /etc/apiserver/specs
+
+COPY demo-vol1.yaml /etc/apiserver/specs/
+COPY apiserver /usr/local/bin/
+COPY maya /usr/local/bin/
+
+COPY entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+ENTRYPOINT entrypoint.sh "${MAYA_API_SERVER_NETWORK}"
+
+EXPOSE 5656

buildscripts/apiserver/build.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+#
+# This script builds the application from source for multiple platforms.
+set -e
+
+# Get the parent directory of where this script is.
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
+DIR="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )"
+
+# Change into that directory
+cd "$DIR"
+
+# Get the git commit
+GIT_COMMIT="$(git rev-parse HEAD)"
+GIT_DIRTY="$(test -n "`git status --porcelain`" && echo "+CHANGES" || true)"
+
+# Determine the arch/os combos we're building for
+XC_ARCH=${XC_ARCH:-"386 amd64"}
+XC_OS=${XC_OS:-"linux"}
+XC_EXCLUDE=${XC_EXCLUDE:-"!darwin/arm !darwin/386"}
+
+# Delete the old contents
+echo "==> Removing old bin/apiserver contents..."
+rm -rf bin/apiserver/*
+mkdir -p bin/apiserver/
+
+# Fetch the tags before using git rev-list --tags
+git fetch --tags >/dev/null 2>&1
+GIT_TAG="$(git describe --tags $(git rev-list --tags --max-count=1))"
+
+if [ -z "${GIT_TAG}" ]; 
+then
+    GIT_TAG="0.0.1"
+fi
+
+if [ -z "${CTLNAME}" ]; 
+then
+    CTLNAME="apiserver"
+fi
+
+# If its dev mode, only build for ourself
+if [[ "${M_APISERVER_DEV}" ]]; then
+    XC_OS=$(go env GOOS)
+    XC_ARCH=$(go env GOARCH)
+fi
+
+# Build!
+echo "==> Building ${CTLNAME} ..."
+
+gox \
+    -os="${XC_OS}" \
+    -arch="${XC_ARCH}" \
+    -osarch="${XC_EXCLUDE}" \
+    -ldflags \
+       "-X main.GitCommit='${GIT_COMMIT}${GIT_DIRTY}' \
+        -X main.CtlName='${CTLNAME}' \
+        -X main.Version='${GIT_TAG}'" \
+    -output "bin/apiserver/{{.OS}}_{{.Arch}}/${CTLNAME}" \
+    .
+
+echo ""
+
+# Move all the compiled things to the $GOPATH/bin
+GOPATH=${GOPATH:-$(go env GOPATH)}
+case $(uname) in
+    CYGWIN*)
+        GOPATH="$(cygpath $GOPATH)"
+        ;;
+esac
+OLDIFS=$IFS
+IFS=: MAIN_GOPATH=($GOPATH)
+IFS=$OLDIFS
+
+# Copy our OS/Arch to ${MAIN_GOPATH}/bin/ directory
+DEV_PLATFORM="./bin/apiserver/$(go env GOOS)_$(go env GOARCH)"
+for F in $(find ${DEV_PLATFORM} -mindepth 1 -maxdepth 1 -type f); do
+    cp ${F} bin/apiserver/
+    cp ${F} ${MAIN_GOPATH}/bin/
+done
+
+if [[ "x${M_APISERVER_DEV}" == "x" ]]; then
+    # Zip and copy to the dist dir
+    echo "==> Packaging..."
+    for PLATFORM in $(find ./bin/apiserver -mindepth 1 -maxdepth 1 -type d); do
+        OSARCH=$(basename ${PLATFORM})
+        echo "--> ${OSARCH}"
+
+        pushd $PLATFORM >/dev/null 2>&1
+        zip ../${CTLNAME}-${OSARCH}.zip ./*
+        popd >/dev/null 2>&1
+    done
+fi
+
+# Done!
+echo
+echo "==> Results:"
+ls -hl bin/apiserver/

buildscripts/apiserver/demo-vol1.yaml

@@ -0,0 +1,6 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: demo-vol1
+  labels:
+    volumeprovisioner.mapi.openebs.io/storage-size: 5G

buildscripts/apiserver/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -ex
+
+MAYA_API_SERVER_NETWORK=$1
+
+CONTAINER_IP_ADDR=$(ip -4 addr show scope global dev "${MAYA_API_SERVER_NETWORK}" | grep inet | awk '{print $2}' | cut -d / -f 1)
+
+# Start apiserver service
+exec /usr/local/bin/apiserver up -bind="${CONTAINER_IP_ADDR}" 1>&2

buildscripts/apiserver/push

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e
+
+IMAGEID=$( sudo docker images -q openebs/m-apiserver:ci )
+
+if [ ! -z "${DNAME}" ] && [ ! -z "${DPASS}" ]; 
+then 
+  sudo docker login -u "${DNAME}" -p "${DPASS}";
+  # Push image to docker hub
+  sudo docker push openebs/m-apiserver:ci ;
+  if [ ! -z "${TRAVIS_TAG}" ] ; 
+  then
+    # Push with different tags if tagged as a release
+    # When github is tagged with a release, then Travis will 
+    # set the release tag in env TRAVIS_TAG
+    sudo docker tag ${IMAGEID} openebs/m-apiserver:${TRAVIS_TAG}
+    sudo docker push openebs/m-apiserver:${TRAVIS_TAG}; 
+    sudo docker tag ${IMAGEID} openebs/m-apiserver:latest
+    sudo docker push openebs/m-apiserver:latest; 
+  fi;
+else
+  echo "No docker credentials provided. Skip uploading openebs/m-apiserver:ci to docker hub"; 
+fi;

orchprovider/k8s/v1/k8s.go

@@ -729,6 +729,54 @@ func (k *k8sOrchestrator) ListStorage(volProProfile volProfile.VolumeProvisioner
 	return pvl, nil
 }
 
+// addNodeTolerationsToDeploy
+func (k *k8sOrchestrator) addNodeTolerationsToDeploy(nodeTaintTolerations []string, deploy *k8sApisExtnsBeta1.Deployment) error {
+
+	// nTT is expected to be in key=value:effect
+	for _, nTT := range nodeTaintTolerations {
+		kveArr := strings.Split(nTT, ":")
+		if len(kveArr) != 2 {
+			return fmt.Errorf("Invalid args '%s' provided for node taint toleration", nTT)
+		}
+
+		kv := kveArr[0]
+		effect := strings.TrimSpace(kveArr[1])
+
+		kvArr := strings.Split(kv, "=")
+		if len(kvArr) != 2 {
+			return fmt.Errorf("Invalid kv '%s' provided for node taint toleration", kv)
+		}
+		k := strings.TrimSpace(kvArr[0])
+		v := strings.TrimSpace(kvArr[1])
+
+		// Setting to blank to validate later
+		e := k8sApiV1.TaintEffect("")
+
+		// Supports only these two effects
+		if string(k8sApiV1.TaintEffectNoExecute) == effect {
+			e = k8sApiV1.TaintEffectNoExecute
+		} else if string(k8sApiV1.TaintEffectNoSchedule) == effect {
+			e = k8sApiV1.TaintEffectNoSchedule
+		}
+
+		if string(e) == "" {
+			return fmt.Errorf("Invalid effect '%s' provided for node taint toleration", effect)
+		}
+
+		toleration := k8sApiV1.Toleration{
+			Key:      k,
+			Operator: k8sApiV1.TolerationOpEqual,
+			Value:    v,
+			Effect:   e,
+		}
+
+		tls := append(deploy.Spec.Template.Spec.Tolerations, toleration)
+		deploy.Spec.Template.Spec.Tolerations = tls
+	}
+
+	return nil
+}
+
 // createControllerDeployment creates a persistent volume controller deployment in
 // kubernetes
 func (k *k8sOrchestrator) createControllerDeployment(volProProfile volProfile.VolumeProvisionerProfile, clusterIP string) (*k8sApisExtnsBeta1.Deployment, error) {
@@ -826,6 +874,19 @@ func (k *k8sOrchestrator) createControllerDeployment(volProProfile volProfile.Vo
 		},
 	}
 
+	// check if node level taint toleration is required ?
+	nTTs, reqd, err := volProProfile.IsControllerNodeTaintTolerations()
+	if err != nil {
+		return nil, err
+	}
+
+	if reqd {
+		err = k.addNodeTolerationsToDeploy(nTTs, deploy)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// add persistent volume controller deployment
 	dd, err := dOps.Create(deploy)
 	if err != nil {
@@ -1022,6 +1083,19 @@ func (k *k8sOrchestrator) createReplicaDeployment(volProProfile volProfile.Volum
 		},
 	}
 
+	// check if node level taint toleration is required ?
+	nTTs, reqd, err := volProProfile.IsReplicaNodeTaintTolerations()
+	if err != nil {
+		return nil, err
+	}
+
+	if reqd {
+		err = k.addNodeTolerationsToDeploy(nTTs, deploy)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	d, err := dOps.Create(deploy)
 	if err != nil {
 		return nil, err

orchprovider/k8s/v1/k8s_test.go

@@ -410,6 +410,11 @@ func (e *okVsmNameVolumeProfile) VSMName() (string, error) {
 	return "ok-vsm-name", nil
 }
 
+// ControllerImage does not return any error
+func (e *okVsmNameVolumeProfile) IsReplicaNodeTaintTolerations() ([]string, bool, error) {
+	return []string{"k=v:NoSchedule"}, true, nil
+}
+
 // okCtrlImgVolumeProfile focusses on not returning any error during invocation
 // of ControllerImage() method
 type okCtrlImgVolumeProfile struct {
@@ -421,6 +426,11 @@ func (e *okCtrlImgVolumeProfile) ControllerImage() (string, bool, error) {
 	return "ok-ctrl-img", true, nil
 }
 
+// ControllerImage does not return any error
+func (e *okCtrlImgVolumeProfile) IsReplicaNodeTaintTolerations() ([]string, bool, error) {
+	return []string{"k=v:NoSchedule"}, true, nil
+}
+
 // errVsmNameVolumeProfile focusses on returning error during invocation of
 // VSMName() method
 type errVsmNameVolumeProfile struct {

types/v1/labels.go

@@ -92,6 +92,21 @@ const (
 	// Usage:
 	// <CTX>_REPLICA_TOPOLOGY_KEY = <some value>
 	PVPReplicaTopologyKeyEnvVarKey EnvironmentVariableKey = "_REPLICA_TOPOLOGY_KEY"
+
+	// PVPControllerNodeTaintTolerationEnvVarKey is the environment variable key
+	// for persistent volume provisioner's node taint toleration
+	//
+	// Usage:
+	// <CTX>_CONTROLLER_NODE_TAINT_TOLERATION = <some value>
+	PVPControllerNodeTaintTolerationEnvVarKey EnvironmentVariableKey = "_CONTROLLER_NODE_TAINT_TOLERATION"
+
+	// PVPReplicaNodeTaintTolerationEnvVarKey is the environment variable key for
+	// persistent volume provisioner's node taint toleration
+	//
+	// Usage:
+	// <CTX>__REPLICA_NODE_TAINT_TOLERATION = <some value>
+	PVPReplicaNodeTaintTolerationEnvVarKey EnvironmentVariableKey = "_REPLICA_NODE_TAINT_TOLERATION"
+
 	// OrchestratorNameEnvVarKey is the environment variable key for
 	// orchestration provider's name
 	//
@@ -231,6 +246,10 @@ const (
 	PVPControllerIPsLbl VolumeProvisionerProfileLabel = "volumeprovisioner.mapi.openebs.io/controller-ips"
 	// Label / Tag for a persistent volume provisioner's persistent path
 	PVPPersistentPathLbl VolumeProvisionerProfileLabel = "volumeprovisioner.mapi.openebs.io/persistent-path"
+	// Label / Tag for a persistent volume provisioner's controller node taint toleration
+	PVPControllerNodeTaintTolerationLbl VolumeProvisionerProfileLabel = "volumeprovisioner.mapi.openebs.io/controller-node-taint-toleration"
+	// Label / Tag for a persistent volume provisioner's replica node taint toleration
+	PVPReplicaNodeTaintTolerationLbl VolumeProvisionerProfileLabel = "volumeprovisioner.mapi.openebs.io/replica-node-taint-toleration"
 
 	// PVPReplicaTopologyKeyLbl is the label for a persistent volume provisioner's
 	// VSM replica topology key