diff --git a/.circleci/config.yml b/.circleci/config.yml index 060927743d..6c451314a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,29 +20,12 @@ jobs: # Run tests working_directory: /test docker: - - image: continuumio/miniconda3 + - image: esmvalgroup/esmvalcore:development steps: - checkout - run: command: | - # Create a file to checksum as cache key - date --rfc-3339 date > cache_key.txt - cat environment.yml >> cache_key.txt - - restore_cache: - key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }} - - run: - # Update/Create Conda environment and run tests - command: | - . /opt/conda/etc/profile.d/conda.sh - # conda update -y conda - conda env update - conda activate esmvaltool python setup.py test --addopts '-m "not installation"' - - save_cache: - key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }} - paths: - - "/opt/conda/envs/esmvaltool" - - ".eggs" - store_test_results: path: test-reports/ - store_artifacts: @@ -137,14 +120,14 @@ jobs: - run: command: | . /opt/conda/etc/profile.d/conda.sh - set -x mkdir /logs # Install # conda update -y conda > /logs/conda.txt 2>&1 conda env update >> /logs/conda.txt 2>&1 - set +x; conda activate esmvaltool; set -x - conda install -yS r-base r-yaml ncl -c conda-forge + conda activate esmvaltool pip install -e .[develop] > /logs/install.txt 2>&1 + # install additional requirements for running all tests + conda install -yS r-base r-yaml ncl -c conda-forge # Log versions dpkg -l > /logs/versions.txt conda env export > /logs/environment.yml diff --git a/doc/quickstart/install.rst b/doc/quickstart/install.rst index cae723895c..baa93946f5 100644 --- a/doc/quickstart/install.rst +++ b/doc/quickstart/install.rst @@ -16,6 +16,99 @@ Once you have installed conda, you can install ESMValCore by running: conda install -c esmvalgroup -c conda-forge esmvalcore +Docker installation +----------------------- + +ESMValCore is also provided through `DockerHub `_ +in the form of docker containers. +See https://docs.docker.com for more information about docker containers and how to +run them. + +You can get the latest release with + +.. code-block:: bash + + docker pull esmvalgroup/esmvalcore:stable + +If you want to use the current master branch, use + +.. code-block:: bash + + docker pull esmvalgroup/esmvalcore:latest + +To run a container using those images, use: + +.. code-block:: bash + + docker run esmvalgroup/esmvalcore:stable --help + +Note that the container does not see the data or environmental variables available in the host by default. +You can make data available with ``-v /path:/path/in/container`` and environmental variables with ``-e VARNAME``. + +For example, the following command would run a recipe + +.. code-block:: bash + + docker run -e HOME -v "$HOME":"$HOME" -v /data:/data esmvalgroup/esmvalcore:stable -c ~/config-user.yml ~/recipes/recipe_example.yml + +with the environmental variable ``$HOME`` available inside the container and the data +in the directories ``$HOME`` and ``/data``, so these can be used to find the configuration file, recipe, and data. + +It might be useful to define a `bash alias +`_ +or script to abbreviate the above command, for example + +.. code-block:: bash + + alias esmvaltool="docker run -e HOME -v $HOME:$HOME -v /data:/data esmvalgroup/esmvalcore:stable" + +would allow using the ``esmvaltool`` command without even noticing that the tool is running inside a Docker container. + + +Singularity installation +---------------------------- + +Docker is usually forbidden in clusters due to security reasons. However, +there is a more secure alternative to run containers that is usually available +on them: `Singularity `_. + +Singularity can use docker containers directly from DockerHub with the +following command + +.. code-block:: bash + + singularity run docker://esmvalgroup/esmvalcore:stable -c ~/config-user.yml ~/recipes/recipe_example.yml + +Note that the container does not see the data available in the host by default. +You can make host data available with ``-B /path:/path/in/container``. + +It might be useful to define a `bash alias +`_ +or script to abbreviate the above command, for example + +.. code-block:: bash + + alias esmvaltool="singularity run -B $HOME:$HOME -B /data:/data docker://esmvalgroup/esmvalcore:stable" + +would allow using the ``esmvaltool`` command without even noticing that the tool is running inside a Singularity container. + +Some clusters may not allow to connect to external services, in those cases +you can first create a singularity image locally: + +.. code-block:: bash + + singularity build esmvalcore.sif docker://esmvalgroup/esmvalcore:stable + +and then upload the image file ``esmvalcore.sif`` to the cluster. +To run the container using the image file ``esmvalcore.sif`` use: + +.. code-block:: bash + + singularity run esmvalcore.sif -c ~/config-user.yml ~/recipes/recipe_example.yml + + + + Development installation ------------------------ @@ -44,4 +137,3 @@ To install from source for development, follow these instructions. e.g. \ ``pip install --trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org -e .[develop]`` - Test that your installation was successful by running ``esmvaltool -h``. - diff --git a/docker/Dockerfile b/docker/Dockerfile index 144134ac63..f0b7a27826 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,16 +1,19 @@ +# To build this container, go to ESMValCore root folder and execute : +# docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile FROM continuumio/miniconda3 # update the conda packages RUN conda update -y conda pip -# install development tools -RUN apt-get update -y && apt-get install -y \ - build-essential \ - curl \ - unzip # install environment packages -RUN conda install -c conda-forge -c esmvalgroup -c birdhouse esmvaltool +COPY . /home/root/source +WORKDIR /home/root/source +RUN ls +RUN conda env update --name base --file environment.yml +RUN pip install . +RUN conda clean --all -y +RUN rm -r /home/root/source # run tests RUN esmvaltool -h diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev new file mode 100644 index 0000000000..446eff005f --- /dev/null +++ b/docker/Dockerfile.dev @@ -0,0 +1,20 @@ +# To build this container, go to ESMValCore root folder and execute : +# docker build -t ${TAG_OF_YOUR_CHOICE} . -f docker/Dockerfile.dev +FROM continuumio/miniconda3 + +# update the conda packages +RUN conda update -y conda pip + +# Copy source +COPY . /home/root/source +WORKDIR /home/root/source + +# Install +RUN apt-get update +RUN conda env update --name base --file environment.yml +RUN pip install -e .[test] +RUN pip uninstall esmvalcore -y + +# Clean up +RUN rm -r /home/root/source +RUN conda clean --all -y diff --git a/docker/docker_guide.md b/docker/docker_guide.md deleted file mode 100644 index ba22ae2cc8..0000000000 --- a/docker/docker_guide.md +++ /dev/null @@ -1,61 +0,0 @@ -# Running ESMValTool packaged in the docker container - -## Install docker -Use official [Docker documentation](https://docs.docker.com/engine/installation/) - -## Get docker image -All available docker images are listed on -[Docker Hub page](https://hub.docker.com/r/esmvalgroup/esmvaltool/tags/) -Before running container you need to pull the image to your machine. - -```sh -docker pull esmvalgroup/esmvaltool:2.0 -``` - -## Running ESMValTool in the container -To run ESMValTool in the docker container you need to mount -necessary directories and provide the name of the namelist -to execute. ESMValTool expects specific directory structure -and directories for input and output files need to be created -in advance. Below are few user scenerios. - -1. Input and output directories are located in the same directory. -```sh -docker run -v :/data/ esmvalgroup/esmvaltool:2.0 nml/.xml -``` -Replace: -* `` with name of the namelist you want to execute -* `` with a location on your local machine -(docker host). It has to be absolute path. - -Content of the `` -directory has to have following structure: -```sh -tree - -├── in # input directory -│ ├── datamodel # model data -│ ├── obsdata # observations -│ └── rawobsdata # raw observations (unprocessed) -└── out # output directory - ├── climo # path for intermediate files (netCDF) - ├── plots # generated plots - ├── regridding # intermediate files generated by regridding process - └── work # output data path (netCDF) -``` - -2. Input and output directories are in different locations. -If input and output directories are in different locations, -they need to be mounted separately. -```sh -docker run \ - -v :/data/in/datamodel \ - -v :/data/in/obsdata \ - -v :/data/in/rawobsdata \ - -v :/data/out/climo \ - -v :/data/out/plots \ - -v :/data/out/regridding \ - -v :/data/out/work \ - esmvalgroup/esmvaltool: nml/.xml -``` -Replace paths in angle brackets `< >` with appropriate values. diff --git a/environment.yml b/environment.yml index f8ac572965..edce7f108d 100644 --- a/environment.yml +++ b/environment.yml @@ -4,10 +4,10 @@ channels: - conda-forge dependencies: + - compilers - esmpy - iris>=2.2.1 - graphviz - - gcc_linux-64 - libunwind # Needed for Python3.7+ - python>=3.6 # if 3.7 lxml will not import correctly if <4.5.0 - python-stratify