Update docker recipe (#603)

.circleci/config.yml

@@ -20,29 +20,12 @@ jobs:
     # Run tests
     working_directory: /test
     docker:
-      - image: continuumio/miniconda3
+      - image: esmvalgroup/esmvalcore:development
     steps:
       - checkout
       - run:
           command: |
-            # Create a file to checksum as cache key
-            date --rfc-3339 date > cache_key.txt
-            cat environment.yml >> cache_key.txt
-      - restore_cache:
-          key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }}
-      - run:
-          # Update/Create Conda environment and run tests
-          command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            # conda update -y conda
-            conda env update
-            conda activate esmvaltool
             python setup.py test --addopts '-m "not installation"'
-      - save_cache:
-          key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }}
-          paths:
-            - "/opt/conda/envs/esmvaltool"
-            - ".eggs"
       - store_test_results:
           path: test-reports/
       - store_artifacts:
@@ -137,14 +120,14 @@ jobs:
       - run:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
-            set -x
             mkdir /logs
             # Install
             # conda update -y conda > /logs/conda.txt 2>&1
             conda env update >> /logs/conda.txt 2>&1
-            set +x; conda activate esmvaltool; set -x
-            conda install -yS r-base r-yaml ncl -c conda-forge
+            conda activate esmvaltool
             pip install -e .[develop] > /logs/install.txt 2>&1
+            # install additional requirements for running all tests
+            conda install -yS r-base r-yaml ncl -c conda-forge
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml

doc/quickstart/install.rst

@@ -16,6 +16,99 @@ Once you have installed conda, you can install ESMValCore by running:
     conda install -c esmvalgroup -c conda-forge esmvalcore
 
 
+Docker installation
+-----------------------
+
+ESMValCore is also provided through `DockerHub <https://hub.docker.com/u/esmvalgroup/>`_
+in the form of docker containers.
+See https://docs.docker.com for more information about docker containers and how to
+run them.
+
+You can get the latest release with
+
+.. code-block:: bash
+
+   docker pull esmvalgroup/esmvalcore:stable
+
+If you want to use the current master branch, use
+
+.. code-block:: bash
+
+   docker pull esmvalgroup/esmvalcore:latest
+
+To run a container using those images, use:
+
+.. code-block:: bash
+
+   docker run esmvalgroup/esmvalcore:stable --help
+
+Note that the container does not see the data or environmental variables available in the host by default.
+You can make data available with ``-v /path:/path/in/container`` and environmental variables with ``-e VARNAME``.
+
+For example, the following command would run a recipe
+
+.. code-block:: bash
+
+   docker run -e HOME -v "$HOME":"$HOME" -v /data:/data esmvalgroup/esmvalcore:stable -c ~/config-user.yml ~/recipes/recipe_example.yml
+
+with the environmental variable ``$HOME`` available inside the container and the data
+in the directories ``$HOME`` and ``/data``, so these can be used to find the configuration file, recipe, and data.
+
+It might be useful to define a `bash alias
+<https://opensource.com/article/19/7/bash-aliases>`_
+or script to abbreviate the above command, for example
+
+.. code-block:: bash
+
+	 alias esmvaltool="docker run -e HOME -v $HOME:$HOME -v /data:/data esmvalgroup/esmvalcore:stable"
+
+would allow using the ``esmvaltool`` command without even noticing that the tool is running inside a Docker container.
+
+
+Singularity installation
+----------------------------
+
+Docker is usually forbidden in clusters due to security reasons. However,
+there is a more secure alternative to run containers that is usually available
+on them: `Singularity <https://sylabs.io/guides/3.0/user-guide/quick_start.html>`_.
+
+Singularity can use docker containers directly from DockerHub with the
+following command
+
+.. code-block:: bash
+
+   singularity run docker://esmvalgroup/esmvalcore:stable -c ~/config-user.yml ~/recipes/recipe_example.yml
+
+Note that the container does not see the data available in the host by default.
+You can make host data available with ``-B /path:/path/in/container``.
+
+It might be useful to define a `bash alias
+<https://opensource.com/article/19/7/bash-aliases>`_
+or script to abbreviate the above command, for example
+
+.. code-block:: bash
+
+	 alias esmvaltool="singularity run -B $HOME:$HOME -B /data:/data docker://esmvalgroup/esmvalcore:stable"
+
+would allow using the ``esmvaltool`` command without even noticing that the tool is running inside a Singularity container.
+
+Some clusters may not allow to connect to external services, in those cases
+you can first create a singularity image locally:
+
+.. code-block:: bash
+
+   singularity build esmvalcore.sif docker://esmvalgroup/esmvalcore:stable
+
+and then upload the image file ``esmvalcore.sif`` to the cluster.
+To run the container using the image file ``esmvalcore.sif`` use:
+
+.. code-block:: bash
+
+   singularity run esmvalcore.sif -c ~/config-user.yml ~/recipes/recipe_example.yml
+
+
+
+
 Development installation
 ------------------------
 
@@ -44,4 +137,3 @@ To install from source for development, follow these instructions.
    e.g. \ ``pip install --trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org -e .[develop]``
 -  Test that your installation was successful by running
    ``esmvaltool -h``.
-

docker/Dockerfile

@@ -1,16 +1,19 @@
+# To build this container, go to ESMValCore root folder and execute :
+# docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile
 FROM continuumio/miniconda3
 
 # update the conda packages
 RUN conda update -y conda pip
 
-# install development tools
-RUN apt-get update -y && apt-get install -y \
-    build-essential \
-    curl \
-    unzip
 
 # install environment packages
-RUN conda install -c conda-forge -c esmvalgroup -c birdhouse esmvaltool
+COPY . /home/root/source
+WORKDIR /home/root/source
+RUN ls
+RUN conda env update --name base --file environment.yml
+RUN pip install .
+RUN conda clean --all -y
+RUN rm -r /home/root/source
 
 # run tests
 RUN esmvaltool -h

docker/Dockerfile.dev

@@ -0,0 +1,20 @@
+# To build this container, go to ESMValCore root folder and execute :
+# docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile.dev
+FROM continuumio/miniconda3
+
+# update the conda packages
+RUN conda update -y conda pip
+
+# Copy source
+COPY . /home/root/source
+WORKDIR /home/root/source
+
+# Install
+RUN apt-get update
+RUN conda env update --name base --file environment.yml
+RUN pip install -e .[test]
+RUN pip uninstall esmvalcore -y
+
+# Clean up
+RUN rm -r /home/root/source
+RUN conda clean --all -y

environment.yml

@@ -4,10 +4,10 @@ channels:
   - conda-forge
 
 dependencies:
+  - compilers
   - esmpy
   - iris>=2.2.1
   - graphviz
-  - gcc_linux-64
   - libunwind  # Needed for Python3.7+
   - python>=3.6  # if 3.7 lxml will not import correctly if <4.5.0
   - python-stratify