diff --git a/HISTORY.rst b/HISTORY.rst index 02a0217af..767e7f044 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,11 +16,15 @@ History * Added ValueEvent class for returning a value with the event * Added methods for waiting on a function to return True * Refactored multitasking classes to use __slots__ for improved memory performance +* Added tools for building yggdrasil docker containers TODO .... * Add tutorial on setting up a service +* Add docs on Docker building +* Add docker tool for creating an executable +* Disable cloning Git repositories based on YAML for services 1.7.0 (2021-08-26) Support for MPI communicators, MPI execution, and pika >= 1.0.0 diff --git a/docs/source/docker.rst b/docs/source/docker.rst new file mode 100644 index 000000000..82967b436 --- /dev/null +++ b/docs/source/docker.rst @@ -0,0 +1,74 @@ +.. _docker_rst: + + +Docker Containers +================= + +For convenience, |yggdrasil| provides `Docker `_ images and tools for building Docker images. To get started using these images, you will need to do a few things first. + +#. **Download and install docker** from `here `_. +#. **Sign-up for DockerHub** `here `_, start docker, and sign-in using your docker hub credentials (either via the desktop app or the `command line `_). + +Release Images +-------------- + +For each tagged release of |yggdrasil|, a Docker image will be published to the ``cropsinsilico/yggdrasil`` `DockerHub repository `_. + +After installing Docker, you can pull the latest release image by running:: + + $ docker pull cropsinsilico/yggdrasil + +or if you need a specific version, ``[VER]``:: + + $ docker pull cropsinsilico/yggdrasil:v[VER] + +You can then run the image as an interactive container:: + + $ docker run -it cropsinsilico/yggdrasil + +If you pulled a specific version, include the version tag in the run command (e.g. ``docker run -it cropsinsilico/yggdrasil:v[VER]``). + +This will start a container using the image and will open a command prompt with container access. You can run |yggdrasil| commands from this prompt. If you would like to run an integration that uses models from your local machine, you will need to mount the model directory as a volume when you launch the container. For example, if you have a model contained in the directory ``/path/to/model``, then you can mount the model as a volume in the container under the directory ``/model_vol`` via:: + + $ docker run -it --volume=/path/to/model:/model_vol cropsinsilico/yggdrasil + +You will then be able to access the model from inside the container and can run integrations using that model. + + +Executable Images +----------------- + +In addition to release images, an executable image will be published for each tagged release of |yggdrasil|. Executable images will be published to the ``cropsinsilico/yggdrasil-executable`` `DockerHub repository `_ and can be pulled via:: + + $ docker pull cropsinsilico/yggdrasil-executable + +Executable images are different than the release images in that they are meant to be treated as an executable and can be used to run integrations using |yggdrasil| without opening the container command line. + +TODO + + +Development Images +------------------ + +Occasionally during development it may be necessary for the |yggdrasil| team to create images for specific commits. These will be published to the ``cropsinsilico/yggdrasil-dev`` `DockerHub repository `_. If you know that such an image exists for a commit with the ID string ``[COMMIT]``, you can pull it via:: + + $ docker pull cropsinsilico/yggdrasil-dev:[COMMIT] + +Such images operate in the same fashion as the release images described above and can be run in the same manner. + + +Building New Images +------------------- + +The ``utils/build_docker.py`` from the |yggdrasil| repository can be used to build/push new Docker images. + + +To build a new Docker image containing the tagged release, ``RELEASE``, run:: + + $ python utils/build_docker.py release RELEASE + +To build a new Docker image containing commit, ``COMMIT``, run:: + + $ python utils/build_docker.py commit COMMIT + +If you add the ``--push`` flag to either of these commands, the image will be pushed to DockerHub after it is built. If you add the ``--executable`` flag, the image will be built such that it exposes the |yggdrasil| CLI and can be used as an executable image in the way described above. diff --git a/docs/source/index.rst b/docs/source/index.rst index eb95f2cd9..a2263bc5b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -24,6 +24,7 @@ Welcome to yggdrasil's documentation! units hpc remote + docker c_format_strings debugging threading diff --git a/utils/build_docker.py b/utils/build_docker.py new file mode 100644 index 000000000..0f98f0377 --- /dev/null +++ b/utils/build_docker.py @@ -0,0 +1,135 @@ +import os +import argparse +import subprocess +_utils_dir = os.path.dirname(__file__) + + +def build(dockerfile, tag, flags=[], repo='cropsinsilico/yggdrasil', + context='.'): + r"""Build a docker image. + + Args: + dockerfile (str): Full path to the docker file that should be used. + tag (str): Tag that should be added to the image. + flags (list, optional): Additional flags that should be passed to + the build command. Defaults to []. + repo (str, optional): DockerHub repository that the image will be + pushed to. Defaults to 'cropsinsilico/yggdrasil'. + context (str, optional): Directory that should be provided as the + context for the image. Defaults to '.'. + + """ + args = ['docker', 'build', '-t', f'{repo}:{tag}', '-f', dockerfile] + flags + args.append(context) + subprocess.call(args) + + +def push_image(tag, repo='cropsinsilico/yggdrasil'): + r"""Push a docker image to DockerHub. + + Args: + tag (str): Tag that should be added to the image. + repo (str, optional): DockerHub repository that the image will be + pushed to. Defaults to 'cropsinsilico/yggdrasil'. + + """ + args = ['docker', 'push', f'{repo}:{tag}'] + subprocess.call(args) + + +def params_release(version): + r"""Build a docker image containing an yggdrasil release. + + Args: + version (str): Release version to install in the image. + + Returns: + dict: Docker build parameters. + + """ + dockerfile = os.path.join(_utils_dir, 'commit.Docker') + # dockerfile = os.path.join(_utils_dir, 'release.Docker') + tag = f'v{version}' + flags = ['--build-arg', f'commit=tags/v{version}'] + # flags = ['--build-arg', f'version={version}'] + repo = 'cropsinsilico/yggdrasil' + return dict(dockerfile=dockerfile, tag=tag, flags=flags, repo=repo) + + +def params_commit(commit): + r"""Build a docker image containing a version of yggdrasil specific to a + commit. + + Args: + commit (str): ID for commit to install from the yggdrasil git repo. + + Returns: + dict: Docker build parameters. + + """ + dockerfile = os.path.join(_utils_dir, 'commit.Docker') + tag = commit + flags = ['--build-arg', f'commit={commit}'] + repo = 'cropsinsilico/yggdrasil-dev' + return dict(dockerfile=dockerfile, tag=tag, flags=flags, repo=repo) + + +def build_executable(params): + r"""Build a docker image containing a version of yggdrasil specific to a + commit or tagged release that can be used as an executable. + + Args: + params (dict): Docker build parameters set based on the base type. + + Returns: + dict: Docker build parameters. + + """ + dockerfile = os.path.join(_utils_dir, 'executable.Docker') + repo = params["tag"] + tag = params["tag"] + flags = ['--build-arg', f'base_image={repo}:{tag}'] + repo = repo.replace('yggdrasil', 'yggdrasil-executable') + return dict(dockerfile=dockerfile, tag=tag, flags=flags, repo=repo) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + "Build a docker image containing a version of yggdrasil.") + subparsers = parser.add_subparsers( + dest="type", + help="Type of docker image that should be built.") + parser_rel = subparsers.add_parser( + "release", help="Build a docker image containing tagged release.") + parser_rel.add_argument( + "version", type=str, + help="Release version that should be installed in the image.") + parser_com = subparsers.add_parser( + "commit", help="Build a docker image containing a specific commit.") + parser_com.add_argument( + "commit", type=str, + help="Commit ID that should be installed in the image.") + joint_args = [ + (("--push", ), + {"action": "store_true", + "help": ("After successfully building the image, push it to " + "DockerHub.")}), + (("--executable", ), + {"action": "store_true", + "help": ("Build the image so that it can be used as an " + "executable.")})] + for iparser in [parser_rel, parser_com]: + for ia, ik in joint_args: + iparser.add_argument(*ia, **ik) + args = parser.parse_args() + if args.type == 'release': + params = params_release(args.version) + elif args.type == 'commit': + params = params_commit(args.commit) + if args.executable: + params = build_executable(params) + dockerfile = params.pop('dockerfile') + tag = params.pop('tag') + build(dockerfile, tag, **params) + if args.push: + push_image(tag, repo=params['repo']) diff --git a/utils/commit.Docker b/utils/commit.Docker new file mode 100644 index 000000000..a83dd2f41 --- /dev/null +++ b/utils/commit.Docker @@ -0,0 +1,34 @@ +FROM continuumio/miniconda3 +SHELL ["/bin/bash", "--login", "-c"] +ARG commit +# RUN apt-get --allow-releaseinfo-change update +RUN apt-get update -y +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda config --add channels conda-forge && \ + conda config --set channel_priority strict + +RUN git clone --recurse-submodules https://github.com/cropsinsilico/yggdrasil.git && \ + cd yggdrasil && \ + git checkout $commit +WORKDIR /yggdrasil +RUN source /opt/conda/etc/profile.d/conda.sh && \ + python utils/setup_test_env.py env --env-name=env conda 3.9 +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda activate env && \ + python utils/setup_test_env.py install conda-dev --always-yes --no-sudo --install-mpi --install-rmq --install-docs --install-sbml --install-omp --python-only + +# Required so that gcc & g++ are available as paths in OpenSimRoot make +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda activate env && \ + # cp $CC $(dirname $CC)/gcc && \ + cp $CXX $(dirname $CXX)/g++ +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda activate env && \ + yggconfig && \ + ygginfo --verbose +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda activate env && \ + yggcompile +RUN echo "conda activate env" > ~/.bashrc + +CMD ["/bin/bash", "--login"] diff --git a/utils/executable.Docker b/utils/executable.Docker new file mode 100644 index 000000000..9998f4586 --- /dev/null +++ b/utils/executable.Docker @@ -0,0 +1,6 @@ +ARG base_image +FROM $base_image + +ENTRYPOINT ["yggdrasil"] + +CMD ["-h"] diff --git a/utils/release.Docker b/utils/release.Docker new file mode 100644 index 000000000..51c3be96b --- /dev/null +++ b/utils/release.Docker @@ -0,0 +1,25 @@ +FROM continuumio/miniconda3 +SHELL ["/bin/bash", "--login", "-c"] +ARG version +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda config --add channels conda-forge && \ + conda config --set channel_priority strict + +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda create -n env python=3.9 yggdrasil=$version + +# Required so that gcc & g++ are available as paths in OpenSimRoot make +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda activate env && \ + cp $CC $(dirname $CC)/gcc && \ + cp $CXX $(dirname $CXX)/g++ +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda activate env && \ + yggconfig && \ + ygginfo --verbose +RUN source /opt/conda/etc/profile.d/conda.sh && \ + conda activate env && \ + yggcompile +RUN echo "conda activate env" > ~/.bashrc + +CMD ["/bin/bash", "--login"] diff --git a/yggdrasil/multitasking.py b/yggdrasil/multitasking.py index e0c6c69c7..191fe8bb0 100644 --- a/yggdrasil/multitasking.py +++ b/yggdrasil/multitasking.py @@ -413,10 +413,7 @@ def __setstate__(self, state): @classmethod def get_base_class(cls, context): r"""Get instance of base class that will be represented.""" - if cls._base_class_name is None: - name = cls.__name__ - else: - name = cls._base_class_name + name = cls._base_class_name context.check_for_base(name) return getattr(context._base, name) diff --git a/yggdrasil/services.py b/yggdrasil/services.py index 05b308402..1fe41333d 100644 --- a/yggdrasil/services.py +++ b/yggdrasil/services.py @@ -652,8 +652,8 @@ def send_request(self, name=None, yamls=None, action='start', **kwargs): yamls = [yamls] if name is None: name = yamls - request = dict(kwargs, name=name, yamls=yamls, action=action, - client_id=self.client_id) + request = dict(kwargs, name=name, yamls=yamls, action=action) + request.setdefault('client_id', self.client_id) wait_for_complete = ((action in ['start', 'stop', 'shutdown']) and (service_type != RMQService)) out = super(IntegrationServiceManager, self).send_request(request) @@ -817,13 +817,18 @@ def integration_info(self, client_id, x): @property def is_running(self): r"""bool: True if the server is running.""" - if not super(IntegrationServiceManager, self).is_running: # pragma: debug - return False - try: - response = self.send_request(action='ping') - return response['status'] == 'running' - except ClientError: + if not super(IntegrationServiceManager, self).is_running: return False + if self.for_request: + try: + response = self.send_request(action='ping') + return response['status'] == 'running' + except ClientError: + return False + else: # pragma: debug + # This would only occur if a server calls is_running while it + # is running (perhaps in a future callback?) + return True def respond(self, request, **kwargs): r"""Create a response to the request. diff --git a/yggdrasil/tests/test_services.py b/yggdrasil/tests/test_services.py index f78f5404f..904c2d8f1 100644 --- a/yggdrasil/tests/test_services.py +++ b/yggdrasil/tests/test_services.py @@ -97,8 +97,11 @@ def running_service(service_type, partial_commtype=None, with_coverage=False): if partial_commtype is not None: lines[-1] += f'commtype=\'{partial_commtype}\'' lines[-1] += ')' - lines += [f'srv.start_server(with_coverage={with_coverage},', - f' log_level={log_level})'] + lines += ['remote_url = srv.address.rstrip(\'/\')', + 'assert(not srv.is_running)', + f'srv.start_server(with_coverage={with_coverage},', + f' log_level={log_level},', + ' remote_url=remote_url)'] with open(script_path, 'w') as fd: fd.write('\n'.join(lines)) args = [sys.executable, script_path] @@ -233,6 +236,7 @@ def test_integration_service(self, running_service): r = requests.get(cli.address) r.raise_for_status() cli.send_request(test_yml, action='status') + cli.send_request(action='status', client_id=None) cli.send_request(test_yml, yamls=test_yml, action='stop') assert_raises(ServerError, cli.send_request, ['invalid'], action='stop') diff --git a/yggdrasil/tests/test_tools.py b/yggdrasil/tests/test_tools.py index 5ac56ebf4..834c2db07 100644 --- a/yggdrasil/tests/test_tools.py +++ b/yggdrasil/tests/test_tools.py @@ -100,6 +100,12 @@ def test_str2bytes(): assert_equal(tools.str2bytes(x, recurse=True), exp) +def test_timer_context(): + r"""Test timer_context.""" + with tools.timer_context("Test timeout: {elapsed}"): + print("timer context body") + + def test_display_source(): r"""Test display_source.""" fname = os.path.abspath(__file__)