Updated docker files (#1651)

* Update docker files * Add esmvaltool install command * Upload coverage for regular unit tests Co-authored-by: Bouwe Andela <[email protected]>
ESMValGroup · Aug 3, 2020 · 346841b · 346841b
1 parent 6cdf84a
commit 346841b
Show file tree

Hide file tree

Showing 12 changed files with 218 additions and 123 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -20,40 +20,33 @@ jobs:
     # Run Python 3 tests
     working_directory: /test
     docker:
-      - image: continuumio/miniconda3
+      - image: esmvalgroup/esmvaltool:development
     steps:
       - checkout
-      - run:
-          command: |
-            # Create a file to checksum as cache key
-            date --rfc-3339 date > cache_key.txt
-            cat environment.yml >> cache_key.txt
-      - restore_cache:
-          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
       - run:
           # Update/Create Conda environment and run tests
           command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            mkdir /logs
-            # conda update -y conda >> /logs/conda.txt 2>&1
-            conda env update >> /logs/conda.txt 2>&1
-            conda activate esmvaltool
-            # Install r-lintr for r linter test
-            Rscript esmvaltool/install/R/setup_devutils.R > /logs/R_install.txt 2>&1
-            # Run tests
-            python setup.py test --addopts '-m "not installation"'
+            julia esmvaltool/install/Julia/setup.jl
+            Rscript esmvaltool/install/R/setup.R
+            python setup.py test
           no_output_timeout: 20m
-      - save_cache:
-          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
-          paths:
-            - "/opt/conda/envs/esmvaltool"
-            - ".eggs"
       - store_artifacts:
           path: /logs
       - store_artifacts:
           path: test-reports/
       - store_test_results:
           path: test-reports/
+      - run:
+          when: always
+          command: |
+            if [[ -v CODACY_PROJECT_TOKEN ]]
+            then
+              echo Uploading coverage report
+              pip install codacy-coverage
+              python-codacy-coverage -r test-reports/coverage.xml
+            else
+              echo Not uploading coverage report
+            fi
 
   test_installation:
     # Test Python 3 installation
@@ -77,15 +70,17 @@ jobs:
             # conda update -y conda > /logs/conda.txt 2>&1
             conda env update >> /logs/conda.txt 2>&1
             set +x; conda activate esmvaltool; set -x
-            pip install . > /logs/install.txt 2>&1
-            Rscript esmvaltool/install/R/setup.R > /logs/R_install.txt 2>&1
-            julia esmvaltool/install/Julia/setup.jl > /logs/julia_install.txt 2>&1
+            pip install .[test] > /logs/install.txt 2>&1
+            esmvaltool install R
+            esmvaltool install Julia
+            # Remove source to test installed software
+            rm -r esmvaltool
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml
             pip freeze > /logs/requirements.txt
             # Test installation
-            python setup.py test
+            pytest
             esmvaltool -h
             ncl -V
             # cdo test, check that it supports hdf5
@@ -105,17 +100,6 @@ jobs:
           path: test-reports/
       - store_test_results:
           path: test-reports/
-      - run:
-          when: always
-          command: |
-            if [[ -v CODACY_PROJECT_TOKEN ]]
-            then
-              echo Uploading coverage report
-              pip install codacy-coverage
-              python-codacy-coverage -r test-reports/coverage.xml
-            else
-              echo Not uploading coverage report
-            fi
 
   develop:
     # Test development installation
@@ -138,8 +122,8 @@ jobs:
             conda env update >> /logs/conda.txt 2>&1
             set +x; conda activate esmvaltool; set -x
             pip install -e .[develop] > /logs/install.txt 2>&1
-            Rscript esmvaltool/install/R/setup.R > /logs/R_install.txt 2>&1
-            julia esmvaltool/install/Julia/setup.jl > /logs/julia_install.txt 2>&1
+            esmvaltool install R
+            esmvaltool install Julia
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml

diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,11 @@
+
+**/__pycache__
+.*
+doc
+package
+tests
+ESMValTool.egg-info
+
+
+!.zenodo.json
+
diff --git a/doc/sphinx/source/community/introduction.rst b/doc/sphinx/source/community/introduction.rst
@@ -51,13 +51,13 @@ To install in development mode, follow these instructions.
    you can declare them with the option ``--trusted-host``, e.g.
    ``pip install --trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org -e .[develop]``
 -  If you want to use R diagnostics, run
-   ``Rscript esmvaltool/install/R/setup.R`` to install the R
+   ``esmvaltool install R`` to install the R
    dependences. Note that if you only want to run the lint test for R
    scripts you will have to install the ``lintr`` package. You can do
    that by running ``Rscript esmvaltool/install/R/setup_devutils.R``.
 -  If you want to use Julia diagnostics, first install Julia as
    described below in section “Installing Julia”, then run
-   ``julia esmvaltool/install/Julia/setup.jl`` to install the Julia
+   ``esmvaltool install Julia`` to install the Julia
    dependences. Install Julia dependences after R dependences if you
    plan to use both.
 -  Test that your installation was succesful by running

diff --git a/doc/sphinx/source/quickstart/installation.rst b/doc/sphinx/source/quickstart/installation.rst
@@ -69,8 +69,96 @@ Note that the ESMValTool source code is contained in the ``esmvaltool-python`` p
 Docker installation
 ===================
 
-.. warning::
-    Docker section to be added
+ESMValTool is also provided through `DockerHub <https://hub.docker.com/u/esmvalgroup/>`_
+in the form of docker containers.
+See https://docs.docker.com for more information about docker containers and how to
+run them.
+
+You can get the latest release with
+
+.. code-block:: bash
+
+   docker pull esmvalgroup/esmvaltool:stable
+
+If you want to use the current master branch, use
+
+.. code-block:: bash
+
+   docker pull esmvalgroup/esmvaltool:latest
+
+To run a container using those images, use:
+
+.. code-block:: bash
+
+   docker run esmvalgroup/esmvaltool:stable --help
+
+Note that the container does not see the data or environmental variables
+available in the host by default. You can make data available with
+``-v /path:/path/in/container`` and environmental variables with ``-e VARNAME``.
+
+For example, the following command would run a recipe
+
+.. code-block:: bash
+
+   docker run -e HOME -v "$HOME":"$HOME" -v /data:/data esmvalgroup/esmvaltool:stable run examples/recipe_python.yml
+
+with the environmental variable ``$HOME`` available inside the container and
+the data in the directories ``$HOME`` and ``/data``, so these can be used to
+find the configuration file, recipe, and data.
+
+It might be useful to define a `bash alias
+<https://opensource.com/article/19/7/bash-aliases>`_
+or script to abbreviate the above command, for example
+
+.. code-block:: bash
+
+	 alias esmvaltool="docker run -e HOME -v $HOME:$HOME -v /data:/data esmvalgroup/esmvaltool:stable"
+
+would allow using the ``esmvaltool`` command without even noticing that the
+tool is running inside a Docker container.
+
+
+Singularity installation
+========================
+
+Docker is usually forbidden in clusters due to security reasons. However,
+there is a more secure alternative to run containers that is usually available
+on them: `Singularity <https://sylabs.io/guides/3.0/user-guide/quick_start.html>`_.
+
+Singularity can use docker containers directly from DockerHub with the
+following command
+
+.. code-block:: bash
+
+   singularity run docker://esmvalgroup/esmvaltool:stable run examples/recipe_python.yml
+
+Note that the container does not see the data available in the host by default.
+You can make host data available with ``-B /path:/path/in/container``.
+
+It might be useful to define a `bash alias
+<https://opensource.com/article/19/7/bash-aliases>`_
+or script to abbreviate the above command, for example
+
+.. code-block:: bash
+
+	 alias esmvaltool="singularity run -B $HOME:$HOME -B /data:/data docker://esmvalgroup/esmvaltool:stable"
+
+would allow using the ``esmvaltool`` command without even noticing that the
+tool is running inside a Singularity container.
+
+Some clusters may not allow to connect to external services, in those cases
+you can first create a singularity image locally:
+
+.. code-block:: bash
+
+   singularity build esmvaltool.sif docker://esmvalgroup/esmvaltool:stable
+
+and then upload the image file ``esmvaltool.sif`` to the cluster.
+To run the container using the image file ``esmvaltool.sif`` use:
+
+.. code-block:: bash
+
+   singularity run esmvaltool.sif run examples/recipe_python.yml
 
 
 Install from source
@@ -207,14 +295,14 @@ If you would like to run Julia diagnostic scripts, you will also need to
 
 .. code-block:: bash
 
-    julia esmvaltool/install/Julia/setup.jl
+    esmvaltool install Julia
 
 If you would like to run R diagnostic scripts, you will also need to install the R
 dependencies. Install the R dependency packages:
 
 .. code-block:: bash
 
-    Rscript esmvaltool/install/R/setup.R
+    esmvaltool install R
 
 The next step is to check that the installation works properly.
 To do this, run the tool with:

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,19 +1,16 @@
+# To build this container, go to ESMValTool root folder and execute:
+# docker build -t esmvaltool:latest . -f docker/Dockerfile
 FROM continuumio/miniconda3
 
-# update the conda packages
-RUN conda update -y conda pip
+RUN apt install julia -y && apt-get clean
+COPY ./environment.yml /src/ESMValTool/environment.yml
+WORKDIR /src/ESMValTool
+RUN conda update -y conda pip && conda env update --name base --file environment.yml && conda clean --all -y
 
-# install development tools
-RUN apt-get update -y && apt-get install -y \
-    build-essential \
-    curl \
-    unzip
+COPY ./esmvaltool/install/R /src/ESMValTool/esmvaltool/install/R
+RUN Rscript ./esmvaltool/install/R/setup.R
 
-# install environment packages
-RUN conda install -c conda-forge -c esmvalgroup -c birdhouse esmvaltool
-
-# run tests
-RUN esmvaltool -h
+COPY . /src/ESMValTool
+RUN pip install -e . && pip cache purge && esmvaltool install Julia
 
 ENTRYPOINT ["esmvaltool"]
-CMD ["-h"]
diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev
@@ -0,0 +1,17 @@
+# To build this container, go to ESMValTool root folder and execute:
+# docker build -t esmvaltool:development . -f docker/Dockerfile.dev
+FROM continuumio/miniconda3
+
+RUN apt install julia -y && apt-get clean
+COPY ./environment.yml /src/ESMValTool/environment.yml
+WORKDIR /src/ESMValTool
+RUN conda update -y conda pip && conda env update --name base --file environment.yml && conda clean --all -y
+
+COPY ./esmvaltool/install/R /src/ESMValTool/esmvaltool/install/R
+RUN Rscript ./esmvaltool/install/R/setup.R
+
+COPY . /src/ESMValTool
+RUN pip install -e .[test] && pip cache purge && esmvaltool install Julia && pip uninstall esmvaltool -y
+RUN rm -rf /src
+
+ENTRYPOINT ["esmvaltool"]
diff --git a/docker/Dockerfile.exp b/docker/Dockerfile.exp
@@ -0,0 +1,16 @@
+# To build this container, go to ESMValTool root folder and execute:
+# docker build -t esmvaltool:experimental . -f docker/Dockerfile.exp
+FROM esmvalgroup/esmvalcore:latest
+
+RUN apt install julia -y && apt-get clean
+COPY ./environment.yml /src/ESMValTool/environment.yml
+WORKDIR /src/ESMValTool
+RUN conda update -y conda pip && conda env update --name base --file environment.yml && conda clean --all -y
+
+COPY ./esmvaltool/install/R /src/ESMValTool/esmvaltool/install/R
+RUN Rscript ./esmvaltool/install/R/setup.R
+
+COPY . /src/ESMValTool
+RUN pip install -e . && pip install ../ESMValCore && pip cache purge && esmvaltool install Julia
+
+ENTRYPOINT ["esmvaltool"]
diff --git a/docker/docker_guide.md b/docker/docker_guide.md
diff --git a/environment.yml b/environment.yml
@@ -7,8 +7,8 @@ channels:
 dependencies:
   # Python packages that cannot be installed from PyPI:
   - gdal
+  - esmpy
   - esmvalcore>=2.0.0,<2.1
-  - esmf
   # Non-Python dependencies
   - cdo>=1.9.7
   - imagemagick
@@ -17,7 +17,6 @@ dependencies:
   - scikit-learn  # may hit hw-specific issue if from pypi https://github.com/scikit-learn/scikit-learn/issues/14485
 
   # Multi language support:
-  - python>=3.6
   - ncl>=6.5.0  # this should always install 6.6.0 though
   - r-base>=3.5
   - r-curl  # Dependency of lintr, but fails to compile because it cannot find libcurl installed from conda.