diff --git a/MANIFEST.in b/MANIFEST.in index e2a888f6..4d330f9d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -21,9 +21,12 @@ include COPYING include *.rst include *.sh +include *.txt include pytest.ini +prune docs/_build recursive-include docs *.py recursive-include docs *.png recursive-include docs *.rst recursive-include docs *.txt +recursive-include scripts reana recursive-include tests *.py diff --git a/docs/index.rst b/docs/index.rst index aca7d99f..e81ddad6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,6 +5,7 @@ :maxdepth: 2 introduction + installation gettingstarted architecture components diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 00000000..cfd2082a --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,86 @@ +Installation +=============== + +Local Kubernetes installation +----------------------------- + +Requirements +```````````` +In order to create a local REANA cluster with Kubernetes as a backend +the following requirements must be met: + +- `VirtualBox` +- `VT-x/AMD-v` virtualization must be enabled in BIOS + +Quickstart +`````````` + +We start by creating a fresh new Python virtual environment that will handle our `REANA` cluster instantiation. + +.. code-block:: console + + $ mkvirtualenv reana + +We update ``pip`` and ``setuptools`` + +.. code-block:: console + + $ pip install --upgrade pip setuptools + +Next we install ``reana`` package + +.. code-block:: console + + $ git clone https://github.com/reanahub/reana + $ cd reana + $ pip install -e .[all] + +Now, for instantiating a local REANA `minikube` based cluster we should run + +.. code-block:: console + + $ reana install-minikube + +which will do the following: + +.. literalinclude:: ../scripts/reana + :language: sh + :lines: 23-89 + +Once the script finishes, we should check ``kubectl get pods`` until we have an output as follows: + +.. code-block:: console + + $ kubectl get pods + job-controller-1390584237-0lt03 1/1 Running 0 1m + message-broker-1410199975-7c5v7 1/1 Running 0 1m + storage-admin 1/1 Running 0 1m + workflow-controller-2689978795-5kzgt 1/1 Running 0 1m + workflow-monitor-1639319062-q7v8z 1/1 Running 0 1m + yadage-alice-worker-1624764635-x8z71 1/1 Running 0 1m + yadage-atlas-worker-2909073811-t9qv3 1/1 Running 0 1m + yadage-cms-worker-209120003-js5cv 1/1 Running 0 1m + yadage-lhcb-worker-4061719987-6gpbn 1/1 Running 0 1m + zeromq-msg-proxy-1617754619-68p7v 1/1 Running 0 1m + +Now we will be able to check whether the nodes are actually working. Firstly, we get the VM's ip: + +.. code-block:: console + + $ export MINIKUBE_IP=$(minikube ip) + +Secondly, we get the current instances ports: + +.. code-block:: console + + $ export WORKFLOW_CONTOLLER_PORT=$(kubectl describe service workflow-controller | grep 'NodePort:' | cut -f 4 | sed -e "s@/TCP@@") + $ export WORKFLOW_MONITOR_PORT=$(kubectl describe service workflow-monitor | grep 'NodePort:' | cut -f 4 | sed -e "s@/TCP@@") + $ export JOB_CONTROLLER_PORT=$(kubectl describe service job-controller | grep 'NodePort:' | cut -f 4 | sed -e "s@/TCP@@") + +And finally, we open the test URLs: + +.. code-block:: console + + $ open "http://$MINIKUBE_IP:$WORKFLOW_CONTROLLER_PORT" + $ open "http://$MINIKUBE_IP:$WORKFLOW_MONITOR_PORT/helloworld" + $ open "http://$MINIKUBE_IP:$JOB_CONTROLLER_PORT/jobs" diff --git a/requirements-k8s.txt b/requirements-k8s.txt new file mode 100644 index 00000000..a6a9bb5c --- /dev/null +++ b/requirements-k8s.txt @@ -0,0 +1 @@ +-e 'git+https://github.com/reanahub/reana-resources-k8s.git@master#egg=reana-resources-k8s' diff --git a/scripts/reana b/scripts/reana new file mode 100755 index 00000000..66b0a051 --- /dev/null +++ b/scripts/reana @@ -0,0 +1,112 @@ +#!/bin/sh +# +# This file is part of REANA. +# Copyright (C) 2017 CERN. +# +# REANA is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# REANA is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# REANA; if not, write to the Free Software Foundation, Inc., 59 Temple Place, +# Suite 330, Boston, MA 02111-1307, USA. +# +# In applying this license, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +# TODO reana-workflow-engine-yadage is specific workflow engine, it shouldn't +# appear in a general installation file. +reana_components="reana-job-controller reana-workflow-controller reana-workflow-monitor + reana-message-broker reana-workflow-engine-yadage" + + +install_local_minikube () { + # Check whether the OS is supported + if ! echo "darwin linux" | grep -F -q -w -i "$(uname -s)" + then + echo 'Your OS is not supported.' + exit 1 + else + OS="$(uname -s | tr '[:upper:]' '[:lower:]')" + fi + + # Install tools `kubectl` and `minikube` if necessary + if ! command -v kubectl > /dev/null 2>&1 + then + echo 'Installing latest kubectl ...' + curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/$OS/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + fi + + if ! command -v minikube > /dev/null 2>&1 + then + echo 'Installing minikube ...' + curl -Lo minikube "https://storage.googleapis.com/minikube/releases/v0.17.1/minikube-$OS-amd64" + chmod +x minikube + sudo mv minikube /usr/local/bin/ + fi + + # FIXME find better way to define this dependency. `extras_require` does not support not PyPI deps and + # `dependency_links` does not seem to work. + pip install -r requirements-k8s.txt + + # Start `minikube` + minikube start || echo 'Error while starting minikube' && exit 1 + + # Use minikube Docker daemon, so local images will be used instead of pulling from registry + echo 'Using minikube Docker daemon ...' + eval "$(minikube docker-env)" + + # Pull all REANA components images and retag for local `dev` + echo 'Pulling REANA components images ...' + for component in $reana_components + do + docker pull "reanahub/$component" + docker tag "reanahub/$component" "$component" + done + + # Generate Kubernetes manifests + echo 'Building REANA configuration manifests ...' + reana-resources-k8s build-manifests + + # Set current cluster secret for `reana-job-controller` component + current_cluster_secret=$(kubectl get secrets | grep 'service-account-token' | cut -d ' ' -f 1) + sed -i'' -e "s/default-token-8p260/$current_cluster_secret/" configuration-manifests/deployments/job-controller.yaml + + # Deploy system + echo 'Deploying REANA on minikube ...' + kubectl create -Rf configuration-manifests + + # Clean configuration manifests + rm -rf configuration-manifests +} + +usage() { + echo "${0##*/} manages REANA system operations" + echo "" + echo "Usage:" + echo " install-minikube Deploys local minikube based REANA cluster" + echo " -h Help" +} + +if [ "$1" != "" ] +then + case $1 in + install-minikube ) install_local_minikube + ;; + clean-minikube) minikube delete + ;; + -h | --help ) usage + exit + ;; + esac +else + usage +fi diff --git a/setup.py b/setup.py index f7ec2cb1..c031085b 100644 --- a/setup.py +++ b/setup.py @@ -84,6 +84,7 @@ extras_require=extras_require, setup_requires=setup_requires, tests_require=tests_require, + scripts=['scripts/reana'], classifiers=[ 'Environment :: Web Environment', 'Intended Audience :: Developers',