diff --git a/GNUmakefile b/GNUmakefile index 5d183c54a3..6c80aba1ad 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -17,8 +17,9 @@ EXTERNAL_TOOLS=\ # list only our .go files i.e. exlcudes any .go files from the vendor directory GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*") -# Specify the name for the maya binary +# Specify the name for the binaries MAYACTL=maya +APISERVER=apiserver # Specify the date o build BUILD_DATE = $(shell date +'%Y%m%d%H%M%S') @@ -29,6 +30,9 @@ dev: format @MAYACTL=${MAYACTL} MAYA_DEV=1 sh -c "'$(PWD)/buildscripts/build.sh'" bin: + @echo "----------------------------" + @echo "--> maya " + @echo "----------------------------" @MAYACTL=${MAYACTL} sh -c "'$(PWD)/buildscripts/build.sh'" initialize: bootstrap @@ -39,6 +43,7 @@ deps: clean: rm -rf bin rm -rf ${GOPATH}/bin/${MAYACTL} + rm -rf ${GOPATH}/bin/${APISERVER} rm -rf ${GOPATH}/pkg/* release: @@ -83,8 +88,9 @@ bootstrap: done image: - @cp bin/maya buildscripts/docker/ + @cp bin/${MAYACTL} buildscripts/docker/ @cd buildscripts/docker && sudo docker build -t openebs/maya:ci --build-arg BUILD_DATE=${BUILD_DATE} . + @rm buildscripts/docker/${MAYACTL} @sh buildscripts/push # You might need to use sudo @@ -97,6 +103,22 @@ maya-agent: # Use this to build only the maya apiserver. apiserver: - GOOS=linux go build ./cmd/apiserver - -.PHONY: all apiserver bin cov integ test vet maya-agent test-nodep + @echo "----------------------------" + @echo "--> apiserver " + @echo "----------------------------" + @CTLNAME=${APISERVER} sh -c "'$(PWD)/buildscripts/apiserver/build.sh'" + +# Currently both mayactl & apiserver binaries are pushed into +# m-apiserver image. This is going to be decoupled soon. +apiserver-image: bin apiserver + @echo "----------------------------" + @echo "--> apiserver image " + @echo "----------------------------" + @cp bin/apiserver/${APISERVER} buildscripts/apiserver/ + @cp bin/${MAYACTL} buildscripts/apiserver/ + @cd buildscripts/apiserver && sudo docker build -t openebs/m-apiserver:ci --build-arg BUILD_DATE=${BUILD_DATE} . + @rm buildscripts/apiserver/${APISERVER} + @rm buildscripts/apiserver/${MAYACTL} + @sh buildscripts/apiserver/push + +.PHONY: all bin cov integ test vet maya-agent test-nodep apiserver apiserver-image diff --git a/Vagrantfile b/Vagrantfile index e02edb3fce..190b0a23c4 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -9,7 +9,7 @@ Vagrant.require_version ">= 1.6.0" M_NODES = ENV['M_NODES'] || 1 # Maya Memory & CPUs -M_MEM = ENV['M_MEM'] || 1024 +M_MEM = ENV['M_MEM'] || 2048 M_CPUS = ENV['M_CPUS'] || 1 # Generic installer script common for server(s) & client(s) @@ -75,7 +75,10 @@ def configureVM(vmCfg, hostname, cpus, mem) vb.cpus = cpus vb.customize ["modifyvm", :id, "--cableconnected1", "on"] end - + + # ensure docker is installed + vmCfg.vm.provision "docker" + # sync your laptop's development with this Vagrant VM vmCfg.vm.synced_folder '.', '/opt/gopath/src/github.com/openebs/maya' diff --git a/buildscripts/apiserver/Dockerfile b/buildscripts/apiserver/Dockerfile new file mode 100644 index 0000000000..4ebbf1536d --- /dev/null +++ b/buildscripts/apiserver/Dockerfile @@ -0,0 +1,39 @@ +# +# This Dockerfile builds a recent maya api server using the latest binary from +# maya api server's releases. +# + +FROM ubuntu:16.04 + +# TODO: The following env variables should be auto detected. +ENV MAYA_API_SERVER_NETWORK="eth0" + +RUN apt-get update && apt-get install -y \ + iproute2 \ + curl \ + net-tools \ + watch \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /etc/apiserver/orchprovider +RUN mkdir -p /etc/apiserver/specs + +COPY demo-vol1.yaml /etc/apiserver/specs/ +COPY apiserver /usr/local/bin/ +COPY maya /usr/local/bin/ + +COPY entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/entrypoint.sh + +ARG BUILD_DATE + +LABEL org.label-schema.name="m-apiserver" +LABEL org.label-schema.description="API server for OpenEBS" +LABEL org.label-schema.url="http://www.openebs.io/" +LABEL org.label-schema.vcs-url="https://github.com/openebs/maya" +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.build-date=$BUILD_DATE + +ENTRYPOINT entrypoint.sh "${MAYA_API_SERVER_NETWORK}" + +EXPOSE 5656 diff --git a/buildscripts/apiserver/build.sh b/buildscripts/apiserver/build.sh new file mode 100755 index 0000000000..4b1b3a4495 --- /dev/null +++ b/buildscripts/apiserver/build.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# +# This script builds the application from source for multiple platforms. +set -e + +# Get the parent directory of where this script is. +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done +DIR="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )" + +# Change into that directory +cd "$DIR" + +# Get the git commit +GIT_COMMIT="$(git rev-parse HEAD)" +GIT_DIRTY="$(test -n "`git status --porcelain`" && echo "+CHANGES" || true)" + +# Determine the arch/os combos we're building for +XC_ARCH=${XC_ARCH:-"386 amd64"} +XC_OS=${XC_OS:-"linux"} +XC_EXCLUDE=${XC_EXCLUDE:-"!darwin/arm !darwin/386"} + +# Delete the old contents +echo "==> Removing old bin/apiserver contents..." +rm -rf bin/apiserver/* +mkdir -p bin/apiserver/ + +# Fetch the tags before using git rev-list --tags +git fetch --tags >/dev/null 2>&1 +GIT_TAG="$(git describe --tags $(git rev-list --tags --max-count=1))" + +if [ -z "${GIT_TAG}" ]; +then + GIT_TAG="0.0.1" +fi + +if [ -z "${CTLNAME}" ]; +then + CTLNAME="apiserver" +fi + +# If its dev mode, only build for ourself +if [[ "${M_APISERVER_DEV}" ]]; then + XC_OS=$(go env GOOS) + XC_ARCH=$(go env GOARCH) +fi + +# Build! +echo "==> Building ${CTLNAME} ..." + +gox \ + -os="${XC_OS}" \ + -arch="${XC_ARCH}" \ + -osarch="${XC_EXCLUDE}" \ + -ldflags \ + "-X main.GitCommit='${GIT_COMMIT}${GIT_DIRTY}' \ + -X main.CtlName='${CTLNAME}' \ + -X main.Version='${GIT_TAG}'" \ + -output "bin/apiserver/{{.OS}}_{{.Arch}}/${CTLNAME}" \ + . + +echo "" + +# Move all the compiled things to the $GOPATH/bin +GOPATH=${GOPATH:-$(go env GOPATH)} +case $(uname) in + CYGWIN*) + GOPATH="$(cygpath $GOPATH)" + ;; +esac +OLDIFS=$IFS +IFS=: MAIN_GOPATH=($GOPATH) +IFS=$OLDIFS + +# Copy our OS/Arch to ${MAIN_GOPATH}/bin/ directory +DEV_PLATFORM="./bin/apiserver/$(go env GOOS)_$(go env GOARCH)" +for F in $(find ${DEV_PLATFORM} -mindepth 1 -maxdepth 1 -type f); do + cp ${F} bin/apiserver/ + cp ${F} ${MAIN_GOPATH}/bin/ +done + +if [[ "x${M_APISERVER_DEV}" == "x" ]]; then + # Zip and copy to the dist dir + echo "==> Packaging..." + for PLATFORM in $(find ./bin/apiserver -mindepth 1 -maxdepth 1 -type d); do + OSARCH=$(basename ${PLATFORM}) + echo "--> ${OSARCH}" + + pushd $PLATFORM >/dev/null 2>&1 + zip ../${CTLNAME}-${OSARCH}.zip ./* + popd >/dev/null 2>&1 + done +fi + +# Done! +echo +echo "==> Results:" +ls -hl bin/apiserver/ diff --git a/buildscripts/apiserver/demo-vol1.yaml b/buildscripts/apiserver/demo-vol1.yaml new file mode 100644 index 0000000000..b96a205c1c --- /dev/null +++ b/buildscripts/apiserver/demo-vol1.yaml @@ -0,0 +1,6 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: demo-vol1 + labels: + volumeprovisioner.mapi.openebs.io/storage-size: 5G diff --git a/buildscripts/apiserver/entrypoint.sh b/buildscripts/apiserver/entrypoint.sh new file mode 100644 index 0000000000..5f87dfa0ed --- /dev/null +++ b/buildscripts/apiserver/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -ex + +MAYA_API_SERVER_NETWORK=$1 + +CONTAINER_IP_ADDR=$(ip -4 addr show scope global dev "${MAYA_API_SERVER_NETWORK}" | grep inet | awk '{print $2}' | cut -d / -f 1) + +# Start apiserver service +exec /usr/local/bin/apiserver up -bind="${CONTAINER_IP_ADDR}" 1>&2 diff --git a/buildscripts/apiserver/push b/buildscripts/apiserver/push new file mode 100755 index 0000000000..a5e27fd0f0 --- /dev/null +++ b/buildscripts/apiserver/push @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +IMAGEID=$( sudo docker images -q openebs/m-apiserver:ci ) + +if [ ! -z "${DNAME}" ] && [ ! -z "${DPASS}" ]; +then + sudo docker login -u "${DNAME}" -p "${DPASS}"; + # Push image to docker hub + sudo docker push openebs/m-apiserver:ci ; + if [ ! -z "${TRAVIS_TAG}" ] ; + then + # Push with different tags if tagged as a release + # When github is tagged with a release, then Travis will + # set the release tag in env TRAVIS_TAG + sudo docker tag ${IMAGEID} openebs/m-apiserver:${TRAVIS_TAG} + sudo docker push openebs/m-apiserver:${TRAVIS_TAG}; + sudo docker tag ${IMAGEID} openebs/m-apiserver:latest + sudo docker push openebs/m-apiserver:latest; + fi; +else + echo "No docker credentials provided. Skip uploading openebs/m-apiserver:ci to docker hub"; +fi; diff --git a/orchprovider/k8s/v1/k8s.go b/orchprovider/k8s/v1/k8s.go index 5488cbdd50..f5cc48826e 100644 --- a/orchprovider/k8s/v1/k8s.go +++ b/orchprovider/k8s/v1/k8s.go @@ -729,6 +729,54 @@ func (k *k8sOrchestrator) ListStorage(volProProfile volProfile.VolumeProvisioner return pvl, nil } +// addNodeTolerationsToDeploy +func (k *k8sOrchestrator) addNodeTolerationsToDeploy(nodeTaintTolerations []string, deploy *k8sApisExtnsBeta1.Deployment) error { + + // nTT is expected to be in key=value:effect + for _, nTT := range nodeTaintTolerations { + kveArr := strings.Split(nTT, ":") + if len(kveArr) != 2 { + return fmt.Errorf("Invalid args '%s' provided for node taint toleration", nTT) + } + + kv := kveArr[0] + effect := strings.TrimSpace(kveArr[1]) + + kvArr := strings.Split(kv, "=") + if len(kvArr) != 2 { + return fmt.Errorf("Invalid kv '%s' provided for node taint toleration", kv) + } + k := strings.TrimSpace(kvArr[0]) + v := strings.TrimSpace(kvArr[1]) + + // Setting to blank to validate later + e := k8sApiV1.TaintEffect("") + + // Supports only these two effects + if string(k8sApiV1.TaintEffectNoExecute) == effect { + e = k8sApiV1.TaintEffectNoExecute + } else if string(k8sApiV1.TaintEffectNoSchedule) == effect { + e = k8sApiV1.TaintEffectNoSchedule + } + + if string(e) == "" { + return fmt.Errorf("Invalid effect '%s' provided for node taint toleration", effect) + } + + toleration := k8sApiV1.Toleration{ + Key: k, + Operator: k8sApiV1.TolerationOpEqual, + Value: v, + Effect: e, + } + + tls := append(deploy.Spec.Template.Spec.Tolerations, toleration) + deploy.Spec.Template.Spec.Tolerations = tls + } + + return nil +} + // createControllerDeployment creates a persistent volume controller deployment in // kubernetes func (k *k8sOrchestrator) createControllerDeployment(volProProfile volProfile.VolumeProvisionerProfile, clusterIP string) (*k8sApisExtnsBeta1.Deployment, error) { @@ -826,6 +874,19 @@ func (k *k8sOrchestrator) createControllerDeployment(volProProfile volProfile.Vo }, } + // check if node level taint toleration is required ? + nTTs, reqd, err := volProProfile.IsControllerNodeTaintTolerations() + if err != nil { + return nil, err + } + + if reqd { + err = k.addNodeTolerationsToDeploy(nTTs, deploy) + if err != nil { + return nil, err + } + } + // add persistent volume controller deployment dd, err := dOps.Create(deploy) if err != nil { @@ -1022,6 +1083,19 @@ func (k *k8sOrchestrator) createReplicaDeployment(volProProfile volProfile.Volum }, } + // check if node level taint toleration is required ? + nTTs, reqd, err := volProProfile.IsReplicaNodeTaintTolerations() + if err != nil { + return nil, err + } + + if reqd { + err = k.addNodeTolerationsToDeploy(nTTs, deploy) + if err != nil { + return nil, err + } + } + d, err := dOps.Create(deploy) if err != nil { return nil, err diff --git a/orchprovider/k8s/v1/k8s_test.go b/orchprovider/k8s/v1/k8s_test.go index da3e172d46..fda49e7ede 100644 --- a/orchprovider/k8s/v1/k8s_test.go +++ b/orchprovider/k8s/v1/k8s_test.go @@ -410,6 +410,11 @@ func (e *okVsmNameVolumeProfile) VSMName() (string, error) { return "ok-vsm-name", nil } +// ControllerImage does not return any error +func (e *okVsmNameVolumeProfile) IsReplicaNodeTaintTolerations() ([]string, bool, error) { + return []string{"k=v:NoSchedule"}, true, nil +} + // okCtrlImgVolumeProfile focusses on not returning any error during invocation // of ControllerImage() method type okCtrlImgVolumeProfile struct { @@ -421,6 +426,11 @@ func (e *okCtrlImgVolumeProfile) ControllerImage() (string, bool, error) { return "ok-ctrl-img", true, nil } +// ControllerImage does not return any error +func (e *okCtrlImgVolumeProfile) IsReplicaNodeTaintTolerations() ([]string, bool, error) { + return []string{"k=v:NoSchedule"}, true, nil +} + // errVsmNameVolumeProfile focusses on returning error during invocation of // VSMName() method type errVsmNameVolumeProfile struct { diff --git a/types/v1/labels.go b/types/v1/labels.go index 811e54d53d..fe6c5ca268 100644 --- a/types/v1/labels.go +++ b/types/v1/labels.go @@ -92,6 +92,21 @@ const ( // Usage: // _REPLICA_TOPOLOGY_KEY = PVPReplicaTopologyKeyEnvVarKey EnvironmentVariableKey = "_REPLICA_TOPOLOGY_KEY" + + // PVPControllerNodeTaintTolerationEnvVarKey is the environment variable key + // for persistent volume provisioner's node taint toleration + // + // Usage: + // _CONTROLLER_NODE_TAINT_TOLERATION = + PVPControllerNodeTaintTolerationEnvVarKey EnvironmentVariableKey = "_CONTROLLER_NODE_TAINT_TOLERATION" + + // PVPReplicaNodeTaintTolerationEnvVarKey is the environment variable key for + // persistent volume provisioner's node taint toleration + // + // Usage: + // __REPLICA_NODE_TAINT_TOLERATION = + PVPReplicaNodeTaintTolerationEnvVarKey EnvironmentVariableKey = "_REPLICA_NODE_TAINT_TOLERATION" + // OrchestratorNameEnvVarKey is the environment variable key for // orchestration provider's name // @@ -231,6 +246,10 @@ const ( PVPControllerIPsLbl VolumeProvisionerProfileLabel = "volumeprovisioner.mapi.openebs.io/controller-ips" // Label / Tag for a persistent volume provisioner's persistent path PVPPersistentPathLbl VolumeProvisionerProfileLabel = "volumeprovisioner.mapi.openebs.io/persistent-path" + // Label / Tag for a persistent volume provisioner's controller node taint toleration + PVPControllerNodeTaintTolerationLbl VolumeProvisionerProfileLabel = "volumeprovisioner.mapi.openebs.io/controller-node-taint-toleration" + // Label / Tag for a persistent volume provisioner's replica node taint toleration + PVPReplicaNodeTaintTolerationLbl VolumeProvisionerProfileLabel = "volumeprovisioner.mapi.openebs.io/replica-node-taint-toleration" // PVPReplicaTopologyKeyLbl is the label for a persistent volume provisioner's // VSM replica topology key diff --git a/types/v1/util.go b/types/v1/util.go index e86f5fc0ee..4af6568b8a 100644 --- a/types/v1/util.go +++ b/types/v1/util.go @@ -516,6 +516,82 @@ func DefaultControllerImage() string { return string(PVPControllerImageDef) } +// GetControllerNodeTaintTolerations gets the node taint tolerations if +// available +func GetControllerNodeTaintTolerations(profileMap map[string]string) (string, error) { + val, err := ControllerNodeTaintTolerations(profileMap) + if err != nil { + return "", err + } + + if val == "" { + val, err = DefaultControllerNodeTaintTolerations() + } + + return val, err +} + +// ControllerNodeTaintTolerations extracts the node taint tolerations +func ControllerNodeTaintTolerations(profileMap map[string]string) (string, error) { + val := "" + if profileMap != nil { + val = strings.TrimSpace(profileMap[string(PVPControllerNodeTaintTolerationLbl)]) + } + + if val != "" { + return val, nil + } + + // else get from environment variable + return OSGetEnv(string(PVPControllerNodeTaintTolerationEnvVarKey), profileMap), nil +} + +// DefaultControllerNodeTaintTolerations will fetch the default value for node +// taint tolerations +func DefaultControllerNodeTaintTolerations() (string, error) { + // Controller node taint toleration property is optional. Hence returns blank + // (i.e. not required) as default. + return "", nil +} + +// GetReplicaNodeTaintTolerations gets the node taint tolerations if +// available +func GetReplicaNodeTaintTolerations(profileMap map[string]string) (string, error) { + val, err := ReplicaNodeTaintTolerations(profileMap) + if err != nil { + return "", err + } + + if val == "" { + val, err = DefaultReplicaNodeTaintTolerations() + } + + return val, err +} + +// ReplicaNodeTaintTolerations extracts the node taint tolerations for replica +func ReplicaNodeTaintTolerations(profileMap map[string]string) (string, error) { + val := "" + if profileMap != nil { + val = strings.TrimSpace(profileMap[string(PVPReplicaNodeTaintTolerationLbl)]) + } + + if val != "" { + return val, nil + } + + // else get from environment variable + return OSGetEnv(string(PVPReplicaNodeTaintTolerationEnvVarKey), profileMap), nil +} + +// DefaultReplicaNodeTaintTolerations will fetch the default value for node +// taint tolerations +func DefaultReplicaNodeTaintTolerations() (string, error) { + // Replica node taint toleration property is optional. Hence returns blank + // (i.e. not required) as default. + return "", nil +} + // GetOrchestratorNetworkType gets the not nil orchestration provider's network // type func GetOrchestratorNetworkType(profileMap map[string]string) string { diff --git a/volumes/profile/volumeprovisioner/profile.go b/volumes/profile/volumeprovisioner/profile.go index e4ae3470ef..66fff259a1 100644 --- a/volumes/profile/volumeprovisioner/profile.go +++ b/volumes/profile/volumeprovisioner/profile.go @@ -71,6 +71,12 @@ type VolumeProvisionerProfile interface { // Get the storage backend i.e. a persistent path of the replica. PersistentPath() (string, error) + + // Verify if node level taint tolerations are required for controller? + IsControllerNodeTaintTolerations() ([]string, bool, error) + + // Verify if node level taint tolerations are required for replica? + IsReplicaNodeTaintTolerations() ([]string, bool, error) } // GetVolProProfileByPVC will return a specific persistent volume provisioner @@ -249,6 +255,48 @@ func (pp *pvcVolProProfile) ReplicaImage() (string, error) { return rImg, nil } +// IsControllerNodeTaintTolerations provides the node level taint tolerations. +// Since node level taint toleration for controller is an optional feature, it +// can return false. +func (pp *pvcVolProProfile) IsControllerNodeTaintTolerations() ([]string, bool, error) { + // Extract the node taint toleration for controller + nTTs, err := v1.GetControllerNodeTaintTolerations(pp.pvc.Labels) + if err != nil { + return nil, false, err + } + + if strings.TrimSpace(nTTs) == "" { + return nil, false, nil + } + + // nTTs is expected of below form + // key=value:effect, key1=value1:effect1 + // __or__ + // key=value:effect + return strings.Split(nTTs, ","), true, nil +} + +// IsReplicaNodeTaintTolerations provides the node level taint tolerations. +// Since node level taint toleration for replica is an optional feature, it +// can return false. +func (pp *pvcVolProProfile) IsReplicaNodeTaintTolerations() ([]string, bool, error) { + // Extract the node taint toleration for replica + nTTs, err := v1.GetReplicaNodeTaintTolerations(pp.pvc.Labels) + if err != nil { + return nil, false, err + } + + if strings.TrimSpace(nTTs) == "" { + return nil, false, nil + } + + // nTTs is expected of below form + // key=value:effect, key1=value1:effect1 + // __or__ + // key=value:effect + return strings.Split(nTTs, ","), true, nil +} + // StorageSize gets the storage size for each persistent volume replica(s) func (pp *pvcVolProProfile) StorageSize() (string, error) { // Extract the storage size from pvc