From 306d045f66c15ade9afe68f5142f1240a95320de Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 16 Mar 2020 11:54:06 +0100 Subject: [PATCH 01/25] Update docker --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 144134ac63..12d5465e35 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update -y && apt-get install -y \ unzip # install environment packages -RUN conda install -c conda-forge -c esmvalgroup -c birdhouse esmvaltool +RUN conda install -c conda-forge -c esmvalgroup esmvalcore # run tests RUN esmvaltool -h From 82b33b98b894633025bdbd4e24ecb315e7c4046f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 1 Apr 2020 11:09:37 +0200 Subject: [PATCH 02/25] Update docker to build from source --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 12d5465e35..b86c6f8c52 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,8 @@ RUN apt-get update -y && apt-get install -y \ unzip # install environment packages -RUN conda install -c conda-forge -c esmvalgroup esmvalcore +RUN conda env update --name base --file environment.yml +RUN pip install . # run tests RUN esmvaltool -h From 6332f45a2cecf720035e8a15b3b2af1e9251aca3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 1 Apr 2020 12:07:30 +0200 Subject: [PATCH 03/25] Update dockerfile --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index b86c6f8c52..a110858515 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,6 +10,8 @@ RUN apt-get update -y && apt-get install -y \ unzip # install environment packages +RUN pwd +RUN ls RUN conda env update --name base --file environment.yml RUN pip install . From 37ed1f7873cec1c9a9d5c1867d9385e79cf8eedd Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 1 Apr 2020 12:25:24 +0200 Subject: [PATCH 04/25] keep testing --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a110858515..d883db0348 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,8 @@ RUN apt-get update -y && apt-get install -y \ unzip # install environment packages -RUN pwd +COPY .. /source +WORKDIR /source RUN ls RUN conda env update --name base --file environment.yml RUN pip install . From 9677ee590570cab0780590679e3bd1d616f46ba4 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 1 Apr 2020 12:37:08 +0200 Subject: [PATCH 05/25] Fix copy --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d883db0348..181dad9530 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update -y && apt-get install -y \ unzip # install environment packages -COPY .. /source +COPY . /source WORKDIR /source RUN ls RUN conda env update --name base --file environment.yml From e4301fe1d663616a87af1d0c2af53213b92fccde Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 6 May 2020 09:30:42 +0200 Subject: [PATCH 06/25] Create dev environment --- docker/Dockerfile.dev | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 docker/Dockerfile.dev diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev new file mode 100644 index 0000000000..190c03b4ab --- /dev/null +++ b/docker/Dockerfile.dev @@ -0,0 +1,17 @@ +FROM continuumio/miniconda3 + +# update the conda packages +RUN conda update -y conda pip + +# install development tools +RUN apt-get update -y && apt-get install -y \ + build-essential \ + curl \ + unzip + +# install environment packages +COPY . /source +WORKDIR /source +RUN ls +RUN conda env update --name base --file environment.yml +RUN pip install -e .[develop] From 1187b3580b86ab74e7ef8a5129d87b3e6eb9dd7d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 6 May 2020 09:31:18 +0200 Subject: [PATCH 07/25] Use dev image for tests --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d6bece157d..08117569e9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,7 +20,7 @@ jobs: # Run tests working_directory: /test docker: - - image: continuumio/miniconda3 + - image: esmvalgroup/esmvalcore:development steps: - checkout - run: @@ -34,8 +34,6 @@ jobs: # Update/Create Conda environment and run tests command: | . /opt/conda/etc/profile.d/conda.sh - # conda update -y conda - conda env update conda activate esmvaltool python setup.py test - save_cache: From 5479293ed185b2e93ffe8b1b7322a9175adb62a1 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 6 May 2020 10:21:10 +0200 Subject: [PATCH 08/25] Update test config --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08117569e9..cf2be7a8a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,8 +33,6 @@ jobs: - run: # Update/Create Conda environment and run tests command: | - . /opt/conda/etc/profile.d/conda.sh - conda activate esmvaltool python setup.py test - save_cache: key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }} From ba999d83628294ffa070998de26fae85909a3431 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 6 May 2020 13:38:51 +0200 Subject: [PATCH 09/25] Remove esmvalcore from dev container --- docker/Dockerfile.dev | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 190c03b4ab..e5c445916f 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -15,3 +15,6 @@ WORKDIR /source RUN ls RUN conda env update --name base --file environment.yml RUN pip install -e .[develop] +RUN pip freeze +RUN pip uninstall esmvalcore +RUN pip freeze From e7a2c0b26ed2a26857a97b681d8fa44508ab514d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 6 May 2020 14:49:29 +0200 Subject: [PATCH 10/25] Add -y to uninstall --- docker/Dockerfile.dev | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index e5c445916f..f87174034f 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -15,6 +15,4 @@ WORKDIR /source RUN ls RUN conda env update --name base --file environment.yml RUN pip install -e .[develop] -RUN pip freeze -RUN pip uninstall esmvalcore -RUN pip freeze +RUN pip uninstall esmvalcore -y From 8775c9cc7c83c0be99b6d6c40a22624da567f4cc Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 6 May 2020 15:02:37 +0200 Subject: [PATCH 11/25] Remove source --- docker/Dockerfile.dev | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index f87174034f..d7c1d68a1f 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -16,3 +16,4 @@ RUN ls RUN conda env update --name base --file environment.yml RUN pip install -e .[develop] RUN pip uninstall esmvalcore -y +RUN rm -r /source From 56cea073b65cecf3847a10419a9adaf58eb1196b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 May 2020 11:24:08 +0200 Subject: [PATCH 12/25] Add clean to docker files --- docker/Dockerfile | 11 ++++------- docker/Dockerfile.dev | 19 +++++++++---------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 181dad9530..1a7903f407 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,18 +3,15 @@ FROM continuumio/miniconda3 # update the conda packages RUN conda update -y conda pip -# install development tools -RUN apt-get update -y && apt-get install -y \ - build-essential \ - curl \ - unzip # install environment packages -COPY . /source -WORKDIR /source +COPY . /home/root/source +WORKDIR /home/root/source RUN ls RUN conda env update --name base --file environment.yml RUN pip install . +RUN conda clean --all -y +RUN rm -r /home/root/source # run tests RUN esmvaltool -h diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index d7c1d68a1f..27a73ff8af 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -3,17 +3,16 @@ FROM continuumio/miniconda3 # update the conda packages RUN conda update -y conda pip -# install development tools -RUN apt-get update -y && apt-get install -y \ - build-essential \ - curl \ - unzip +# Copy source +COPY . /home/root/source +WORKDIR /home/root/source -# install environment packages -COPY . /source -WORKDIR /source -RUN ls +# Install +RUN apt-get update -y && apt-get install -y build-essential RUN conda env update --name base --file environment.yml RUN pip install -e .[develop] RUN pip uninstall esmvalcore -y -RUN rm -r /source + +# Clean up +RUN rm -r /home/root/source +RUN conda clean --all -y From bb665b7f75640a18d7076e2ff8916d9e14e31a46 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 May 2020 11:51:22 +0200 Subject: [PATCH 13/25] Add comment on how to build in dockerfiles --- docker/Dockerfile | 2 ++ docker/Dockerfile.dev | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1a7903f407..f0b7a27826 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,5 @@ +# To build this container, go to ESMValCore root folder and execute : +# docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile FROM continuumio/miniconda3 # update the conda packages diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 27a73ff8af..003e01ba09 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,3 +1,5 @@ +# To build this container, go to ESMValCore root folder and execute : +# docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile.dev FROM continuumio/miniconda3 # update the conda packages From 5dcd611aa9b64542acb6ebcc4b5deab53f0cc007 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 May 2020 12:12:34 +0200 Subject: [PATCH 14/25] Add installation with docker in doc --- doc/quickstart/install.rst | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/doc/quickstart/install.rst b/doc/quickstart/install.rst index cae723895c..8e19aa20e8 100644 --- a/doc/quickstart/install.rst +++ b/doc/quickstart/install.rst @@ -16,6 +16,33 @@ Once you have installed conda, you can install ESMValCore by running: conda install -c esmvalgroup -c conda-forge esmvalcore +Using it through docker +----------------------- + +ESMValCore is also provided thorugh DockerHub in the form of docker containers. +See https://docs.docker.com/ for more info about docker containers and how to +run the + +You can get the latest release with + +.. code-block:: bash + docker pull esmvalgroup/esmvalcore:stable + +If yoy want to use the current master branch, use + +.. code-block:: bash + docker pull esmvalgroup/esmvalcore:latest + +To run a container using those images, use: + +.. code-block:: bash + docker run esmvalgroup/esmvalcore:stable esmvaltool -v + +.. warning:: + The container does not see the data available in the host by default. + You can make host data available with `-v /path:/path/in/container` + + Development installation ------------------------ From 83a2b7f4c2879dbdd752a6697460d6beeeba4095 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 May 2020 12:13:57 +0200 Subject: [PATCH 15/25] Update circleci --- .circleci/config.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d2c0fde50..eca15a83be 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,22 +23,9 @@ jobs: - image: esmvalgroup/esmvalcore:development steps: - checkout - - run: - command: | - # Create a file to checksum as cache key - date --rfc-3339 date > cache_key.txt - cat environment.yml >> cache_key.txt - - restore_cache: - key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }} - run: command: | python setup.py test --addopts '-m "not installation"' - - - save_cache: - key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }} - paths: - - "/opt/conda/envs/esmvaltool" - - ".eggs" - store_test_results: path: test-reports/ - store_artifacts: From 0bfc284839becd0bb4e4bceb3cf3923ecec63fc4 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 May 2020 12:26:24 +0200 Subject: [PATCH 16/25] Fix doc --- doc/quickstart/install.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/quickstart/install.rst b/doc/quickstart/install.rst index 8e19aa20e8..5046e6c0f2 100644 --- a/doc/quickstart/install.rst +++ b/doc/quickstart/install.rst @@ -26,19 +26,23 @@ run the You can get the latest release with .. code-block:: bash + docker pull esmvalgroup/esmvalcore:stable If yoy want to use the current master branch, use .. code-block:: bash + docker pull esmvalgroup/esmvalcore:latest To run a container using those images, use: .. code-block:: bash + docker run esmvalgroup/esmvalcore:stable esmvaltool -v .. warning:: + The container does not see the data available in the host by default. You can make host data available with `-v /path:/path/in/container` From 11ce28b123a6fbb69132fdbe4462093b54835d98 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 20 May 2020 15:57:34 +0200 Subject: [PATCH 17/25] Update dev environment --- docker/Dockerfile.dev | 4 ++-- environment.yml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 003e01ba09..446eff005f 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -10,9 +10,9 @@ COPY . /home/root/source WORKDIR /home/root/source # Install -RUN apt-get update -y && apt-get install -y build-essential +RUN apt-get update RUN conda env update --name base --file environment.yml -RUN pip install -e .[develop] +RUN pip install -e .[test] RUN pip uninstall esmvalcore -y # Clean up diff --git a/environment.yml b/environment.yml index aa4772edec..7e755adbe2 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,6 @@ dependencies: - esmpy - iris>=2.2.1 - graphviz - - gcc_linux-64 - libunwind # Needed for Python3.7+ - python>=3.6 - python-stratify From 1f616c143b2f9e7eab60b679aabe8e975b94a43c Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 20 May 2020 20:34:07 +0200 Subject: [PATCH 18/25] Check if pip install .[develop] works with dependencies from environment.yml Change linux compilers to OS independent compilers --- .circleci/config.yml | 6 +++--- environment.yml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eca15a83be..6c451314a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,14 +120,14 @@ jobs: - run: command: | . /opt/conda/etc/profile.d/conda.sh - set -x mkdir /logs # Install # conda update -y conda > /logs/conda.txt 2>&1 conda env update >> /logs/conda.txt 2>&1 - set +x; conda activate esmvaltool; set -x - conda install -yS r-base r-yaml ncl -c conda-forge + conda activate esmvaltool pip install -e .[develop] > /logs/install.txt 2>&1 + # install additional requirements for running all tests + conda install -yS r-base r-yaml ncl -c conda-forge # Log versions dpkg -l > /logs/versions.txt conda env export > /logs/environment.yml diff --git a/environment.yml b/environment.yml index 7e755adbe2..576d5c831c 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,7 @@ channels: - conda-forge dependencies: + - compilers - esmpy - iris>=2.2.1 - graphviz From d62caa546001caef67f7967969f04609222b182e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 21 May 2020 11:09:15 +0200 Subject: [PATCH 19/25] Use alpine based image --- docker/Dockerfile | 2 +- docker/Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f0b7a27826..153122aad8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ # To build this container, go to ESMValCore root folder and execute : # docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile -FROM continuumio/miniconda3 +FROM continuumio/miniconda3:4.8.2-alpine # update the conda packages RUN conda update -y conda pip diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 446eff005f..48665afe80 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,6 +1,6 @@ # To build this container, go to ESMValCore root folder and execute : # docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile.dev -FROM continuumio/miniconda3 +FROM continuumio/miniconda3:4.8.2-alpine # update the conda packages RUN conda update -y conda pip From 03e482379db7cb9511e8f3ddc59551980d84a737 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 21 May 2020 11:21:06 +0200 Subject: [PATCH 20/25] use previous conda version --- docker/Dockerfile | 2 +- docker/Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 153122aad8..c332974ca1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ # To build this container, go to ESMValCore root folder and execute : # docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile -FROM continuumio/miniconda3:4.8.2-alpine +FROM continuumio/miniconda3:4.7.12 # update the conda packages RUN conda update -y conda pip diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 48665afe80..3ed017ce87 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,6 +1,6 @@ # To build this container, go to ESMValCore root folder and execute : # docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile.dev -FROM continuumio/miniconda3:4.8.2-alpine +FROM continuumio/miniconda3:4.7.12 # update the conda packages RUN conda update -y conda pip From dece323dd7877f061e0de8b288c62002f2461b54 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 21 May 2020 13:23:41 +0200 Subject: [PATCH 21/25] Update docker recipe --- docker/Dockerfile | 2 +- docker/Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c332974ca1..f0b7a27826 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ # To build this container, go to ESMValCore root folder and execute : # docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile -FROM continuumio/miniconda3:4.7.12 +FROM continuumio/miniconda3 # update the conda packages RUN conda update -y conda pip diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 3ed017ce87..446eff005f 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,6 +1,6 @@ # To build this container, go to ESMValCore root folder and execute : # docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile.dev -FROM continuumio/miniconda3:4.7.12 +FROM continuumio/miniconda3 # update the conda packages RUN conda update -y conda pip From faa7cd37ce3f87df355225eed7783f923c66ed2a Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 26 May 2020 10:51:55 +0200 Subject: [PATCH 22/25] Add singularity info --- doc/quickstart/install.rst | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/doc/quickstart/install.rst b/doc/quickstart/install.rst index 5046e6c0f2..eb83b9446d 100644 --- a/doc/quickstart/install.rst +++ b/doc/quickstart/install.rst @@ -16,7 +16,7 @@ Once you have installed conda, you can install ESMValCore by running: conda install -c esmvalgroup -c conda-forge esmvalcore -Using it through docker +Using it through Docker ----------------------- ESMValCore is also provided thorugh DockerHub in the form of docker containers. @@ -46,6 +46,40 @@ To run a container using those images, use: The container does not see the data available in the host by default. You can make host data available with `-v /path:/path/in/container` +Using it through Singularity +---------------------------- + +Docker is usually forbidden in clusters due to security reasons. However, +there is a more secure alternative to run containers that is usually available +on them: Singularity. + +Singularity can use docker containers directly from DockerHub with the +following command + +.. code-block:: bash + + singularity run docker://esmvalgroup/esmvalcore:${DESIRED_TAG} -c ${PATH_TO_CONFIG_FILE} ${PATH_TO_RECIPE} + +.. warning:: + + As with docker, The container does not see the data available in the host by default. + You can make host data available with `-B /path:/path/in/container` + +Some clusters may not allow to connect to external services, in those cases +you can create a singularity image locally that can later be uploaded to the cluster. + +.. code-block:: bash + + singularity build esmvalcore_${TAG}.sif docker://esmvalgroup/esmvalcore:${TAG} + +To run the container using the newly created image file use: + +.. code-block:: bash + + singularity run esmvalcore_${TAG}.sif -c ${PATH_TO_CONFIG_FILE} ${PATH_TO_RECIPE} + + + Development installation ------------------------ From cdec592340790dd66f41abde03ff73ebb82b2d64 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 26 May 2020 12:10:55 +0200 Subject: [PATCH 23/25] Apply suggestions from code review Co-authored-by: Bouwe Andela --- doc/quickstart/install.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/doc/quickstart/install.rst b/doc/quickstart/install.rst index eb83b9446d..0d0686046e 100644 --- a/doc/quickstart/install.rst +++ b/doc/quickstart/install.rst @@ -16,12 +16,12 @@ Once you have installed conda, you can install ESMValCore by running: conda install -c esmvalgroup -c conda-forge esmvalcore -Using it through Docker +Docker installation ----------------------- -ESMValCore is also provided thorugh DockerHub in the form of docker containers. +ESMValCore is also provided through DockerHub in the form of docker containers. See https://docs.docker.com/ for more info about docker containers and how to -run the +run them. You can get the latest release with @@ -29,7 +29,7 @@ You can get the latest release with docker pull esmvalgroup/esmvalcore:stable -If yoy want to use the current master branch, use +If you want to use the current master branch, use .. code-block:: bash @@ -41,12 +41,12 @@ To run a container using those images, use: docker run esmvalgroup/esmvalcore:stable esmvaltool -v -.. warning:: +.. note:: - The container does not see the data available in the host by default. + The container does not see the data or environmental variables available in the host by default. You can make host data available with `-v /path:/path/in/container` -Using it through Singularity +Singularity installation ---------------------------- Docker is usually forbidden in clusters due to security reasons. However, @@ -60,7 +60,7 @@ following command singularity run docker://esmvalgroup/esmvalcore:${DESIRED_TAG} -c ${PATH_TO_CONFIG_FILE} ${PATH_TO_RECIPE} -.. warning:: +.. note:: As with docker, The container does not see the data available in the host by default. You can make host data available with `-B /path:/path/in/container` @@ -109,4 +109,3 @@ To install from source for development, follow these instructions. e.g. \ ``pip install --trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org -e .[develop]`` - Test that your installation was successful by running ``esmvaltool -h``. - From 5c542ccdcd392b3dea60fd6c98b08a93c57139e1 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 8 Jun 2020 18:02:47 +0200 Subject: [PATCH 24/25] Add examples and more links --- doc/quickstart/install.rst | 62 +++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/doc/quickstart/install.rst b/doc/quickstart/install.rst index 0d0686046e..baa93946f5 100644 --- a/doc/quickstart/install.rst +++ b/doc/quickstart/install.rst @@ -19,8 +19,9 @@ Once you have installed conda, you can install ESMValCore by running: Docker installation ----------------------- -ESMValCore is also provided through DockerHub in the form of docker containers. -See https://docs.docker.com/ for more info about docker containers and how to +ESMValCore is also provided through `DockerHub `_ +in the form of docker containers. +See https://docs.docker.com for more information about docker containers and how to run them. You can get the latest release with @@ -39,44 +40,71 @@ To run a container using those images, use: .. code-block:: bash - docker run esmvalgroup/esmvalcore:stable esmvaltool -v + docker run esmvalgroup/esmvalcore:stable --help -.. note:: +Note that the container does not see the data or environmental variables available in the host by default. +You can make data available with ``-v /path:/path/in/container`` and environmental variables with ``-e VARNAME``. + +For example, the following command would run a recipe + +.. code-block:: bash + + docker run -e HOME -v "$HOME":"$HOME" -v /data:/data esmvalgroup/esmvalcore:stable -c ~/config-user.yml ~/recipes/recipe_example.yml + +with the environmental variable ``$HOME`` available inside the container and the data +in the directories ``$HOME`` and ``/data``, so these can be used to find the configuration file, recipe, and data. + +It might be useful to define a `bash alias +`_ +or script to abbreviate the above command, for example + +.. code-block:: bash + + alias esmvaltool="docker run -e HOME -v $HOME:$HOME -v /data:/data esmvalgroup/esmvalcore:stable" + +would allow using the ``esmvaltool`` command without even noticing that the tool is running inside a Docker container. - The container does not see the data or environmental variables available in the host by default. - You can make host data available with `-v /path:/path/in/container` Singularity installation ---------------------------- -Docker is usually forbidden in clusters due to security reasons. However, -there is a more secure alternative to run containers that is usually available -on them: Singularity. +Docker is usually forbidden in clusters due to security reasons. However, +there is a more secure alternative to run containers that is usually available +on them: `Singularity `_. Singularity can use docker containers directly from DockerHub with the following command .. code-block:: bash - singularity run docker://esmvalgroup/esmvalcore:${DESIRED_TAG} -c ${PATH_TO_CONFIG_FILE} ${PATH_TO_RECIPE} + singularity run docker://esmvalgroup/esmvalcore:stable -c ~/config-user.yml ~/recipes/recipe_example.yml + +Note that the container does not see the data available in the host by default. +You can make host data available with ``-B /path:/path/in/container``. + +It might be useful to define a `bash alias +`_ +or script to abbreviate the above command, for example + +.. code-block:: bash -.. note:: + alias esmvaltool="singularity run -B $HOME:$HOME -B /data:/data docker://esmvalgroup/esmvalcore:stable" - As with docker, The container does not see the data available in the host by default. - You can make host data available with `-B /path:/path/in/container` +would allow using the ``esmvaltool`` command without even noticing that the tool is running inside a Singularity container. Some clusters may not allow to connect to external services, in those cases -you can create a singularity image locally that can later be uploaded to the cluster. +you can first create a singularity image locally: .. code-block:: bash - singularity build esmvalcore_${TAG}.sif docker://esmvalgroup/esmvalcore:${TAG} + singularity build esmvalcore.sif docker://esmvalgroup/esmvalcore:stable -To run the container using the newly created image file use: +and then upload the image file ``esmvalcore.sif`` to the cluster. +To run the container using the image file ``esmvalcore.sif`` use: .. code-block:: bash - singularity run esmvalcore_${TAG}.sif -c ${PATH_TO_CONFIG_FILE} ${PATH_TO_RECIPE} + singularity run esmvalcore.sif -c ~/config-user.yml ~/recipes/recipe_example.yml From 555c1e6691815266d63e91b7a929a2e154bae6d1 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 19 Jun 2020 14:33:22 +0200 Subject: [PATCH 25/25] Remove outdated guide --- docker/docker_guide.md | 61 ------------------------------------------ 1 file changed, 61 deletions(-) delete mode 100644 docker/docker_guide.md diff --git a/docker/docker_guide.md b/docker/docker_guide.md deleted file mode 100644 index ba22ae2cc8..0000000000 --- a/docker/docker_guide.md +++ /dev/null @@ -1,61 +0,0 @@ -# Running ESMValTool packaged in the docker container - -## Install docker -Use official [Docker documentation](https://docs.docker.com/engine/installation/) - -## Get docker image -All available docker images are listed on -[Docker Hub page](https://hub.docker.com/r/esmvalgroup/esmvaltool/tags/) -Before running container you need to pull the image to your machine. - -```sh -docker pull esmvalgroup/esmvaltool:2.0 -``` - -## Running ESMValTool in the container -To run ESMValTool in the docker container you need to mount -necessary directories and provide the name of the namelist -to execute. ESMValTool expects specific directory structure -and directories for input and output files need to be created -in advance. Below are few user scenerios. - -1. Input and output directories are located in the same directory. -```sh -docker run -v :/data/ esmvalgroup/esmvaltool:2.0 nml/.xml -``` -Replace: -* `` with name of the namelist you want to execute -* `` with a location on your local machine -(docker host). It has to be absolute path. - -Content of the `` -directory has to have following structure: -```sh -tree - -├── in # input directory -│ ├── datamodel # model data -│ ├── obsdata # observations -│ └── rawobsdata # raw observations (unprocessed) -└── out # output directory - ├── climo # path for intermediate files (netCDF) - ├── plots # generated plots - ├── regridding # intermediate files generated by regridding process - └── work # output data path (netCDF) -``` - -2. Input and output directories are in different locations. -If input and output directories are in different locations, -they need to be mounted separately. -```sh -docker run \ - -v :/data/in/datamodel \ - -v :/data/in/obsdata \ - -v :/data/in/rawobsdata \ - -v :/data/out/climo \ - -v :/data/out/plots \ - -v :/data/out/regridding \ - -v :/data/out/work \ - esmvalgroup/esmvaltool: nml/.xml -``` -Replace paths in angle brackets `< >` with appropriate values.