From d68d88c46f6f23ce3f79dff2bef4fe978de10b1e Mon Sep 17 00:00:00 2001 From: Diego Rodriguez Date: Mon, 6 Feb 2017 13:35:40 +0100 Subject: [PATCH] docs: fix ReadTheDocs build * Adds `config.py` so that pykube configuration is loaded only at run time. (addresses #18) * Renames `create_job` on `k8s.py` to avoid name clashing inside `app.py`. Signed-off-by: Diego Rodriguez --- MANIFEST.in | 1 + config.py | 29 +++++++++++++++++++++++++++++ docs/conf.py | 10 ++++++---- reana_job_controller/app.py | 28 ++++++++++++++++------------ reana_job_controller/k8s.py | 22 ++++++++++++---------- 5 files changed, 64 insertions(+), 26 deletions(-) create mode 100644 config.py diff --git a/MANIFEST.in b/MANIFEST.in index e42fd44a..b53d78d1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -22,6 +22,7 @@ include Dockerfile include COPYING include *.rst include *.sh +include *.py include pytest.ini recursive-include reana_job_controller *.json recursive-include docs *.py diff --git a/config.py b/config.py new file mode 100644 index 00000000..54751af0 --- /dev/null +++ b/config.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2017 CERN. +# +# REANA is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# REANA is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# REANA; if not, write to the Free Software Foundation, Inc., 59 Temple Place, +# Suite 330, Boston, MA 02111-1307, USA. +# +# In applying this license, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +"""Flask application configuration.""" + +import pykube + +# FIXME do not share HTTPClient but only config +PYKUBE_API = pykube.HTTPClient(pykube.KubeConfig.from_service_account()) +PYKUBE_API.session.verify = False diff --git a/docs/conf.py b/docs/conf.py index e43e055f..30a1f98c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,12 @@ from __future__ import print_function import os -import sphinx.environment +import sys + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('..')) # -- General configuration ------------------------------------------------ @@ -196,6 +201,3 @@ author, 'reana', 'One line description of project.', 'Miscellaneous'), ] - -# Mock autodoc imports: -autodoc_mock_imports = ['k8s', ] diff --git a/reana_job_controller/app.py b/reana_job_controller/app.py index 16294f77..ccc618f4 100644 --- a/reana_job_controller/app.py +++ b/reana_job_controller/app.py @@ -29,7 +29,7 @@ from flask import Flask, abort, jsonify, request -import k8s +from reana_job_controller.k8s import instantiate_job, watch_jobs, watch_pods app = Flask(__name__) app.secret_key = "mega secret key" @@ -179,13 +179,13 @@ def create_job(): job_id = str(uuid.uuid4()) - job_obj = k8s.create_job(job_id, - request.json['docker-img'], - cmd, - cvmfs_repos, - env_vars, - request.json['experiment'], - shared_file_system=True) + job_obj = instantiate_job(job_id, + request.json['docker-img'], + cmd, + cvmfs_repos, + env_vars, + request.json['experiment'], + shared_file_system=True) if job_obj: job = copy.deepcopy(request.json) @@ -265,11 +265,15 @@ def get_job(job_id): level=logging.DEBUG, format='%(asctime)s - %(threadName)s - %(levelname)s: %(message)s' ) - job_event_reader_thread = threading.Thread(target=k8s.watch_jobs, - args=(JOB_DB,)) + app.config.from_object('config') + + job_event_reader_thread = threading.Thread(target=watch_jobs, + args=(JOB_DB, + app.config['PYKUBE_API'])) job_event_reader_thread.start() - pod_event_reader_thread = threading.Thread(target=k8s.watch_pods, - args=(JOB_DB,)) + pod_event_reader_thread = threading.Thread(target=watch_pods, + args=(JOB_DB, + app.config['PYKUBE_API'])) pod_event_reader_thread.start() app.run(debug=True, port=5000, host='0.0.0.0') diff --git a/reana_job_controller/k8s.py b/reana_job_controller/k8s.py index ac35138f..d7fc2628 100644 --- a/reana_job_controller/k8s.py +++ b/reana_job_controller/k8s.py @@ -24,11 +24,11 @@ import logging import time + import pykube -import volume_templates +from flask import current_app as app -api = pykube.HTTPClient(pykube.KubeConfig.from_service_account()) -api.session.verify = False +import reana_job_controller.volume_templates def add_shared_volume(job, namespace): @@ -46,8 +46,8 @@ def add_shared_volume(job, namespace): job['spec']['template']['spec']['volumes'].append(volume) -def create_job(job_id, docker_img, cmd, cvmfs_repos, env_vars, namespace, - shared_file_system): +def instantiate_job(job_id, docker_img, cmd, cvmfs_repos, env_vars, namespace, + shared_file_system): """Create Kubernetes job. :param job_id: Job uuid. @@ -119,21 +119,22 @@ def create_job(job_id, docker_img, cmd, cvmfs_repos, env_vars, namespace, # add better handling try: - job_obj = pykube.Job(api, job) + job_obj = pykube.Job(app.config['PYKUBE_API'], job) job_obj.create() return job_obj except pykube.exceptions.HTTPError: return None -def watch_jobs(job_db): +def watch_jobs(job_db, api_client): """Open stream connection to k8s apiserver to watch all jobs status. :param job_db: Dictionary which contains all current jobs. """ while True: logging.debug('Starting a new stream request to watch Jobs') - stream = pykube.Job.objects(api).filter(namespace=pykube.all).watch() + stream = pykube.Job.objects( + api_client).filter(namespace=pykube.all).watch() for event in stream: logging.info('New Job event received') job = event.object @@ -177,14 +178,15 @@ def watch_jobs(job_db): job.delete() -def watch_pods(job_db): +def watch_pods(job_db, api_client): """Open stream connection to k8s apiserver to watch all pods status. :param job_db: Dictionary which contains all current jobs. """ while True: logging.info('Starting a new stream request to watch Pods') - stream = pykube.Pod.objects(api).filter(namespace=pykube.all).watch() + stream = pykube.Pod.objects( + api_client).filter(namespace=pykube.all).watch() for event in stream: logging.info('New Pod event received') pod = event.object