Merge remote-tracking branch 'upstream/v2.0.x'

* upstream/v2.0.x:
  Improved the way requirements are handled, ensuring that pip gets all the information it needs. (#2890)

.travis.yml

@@ -24,80 +24,97 @@ env:
     - secure: "N9/qBUT5CqfC7KQBDy5mIWZcGNuUJk3e/qmKJpotWYV+zwOI4GghJsRce6nFnlRiwl65l5oBEcvf3+sBvUfbZqh7U0MdHpw2tHhr2FSCmMB3bkvARZblh9M37f4da9G9VmRkqnyBM5G5TImXtoq4dusvNWKvLW0qETciaipq7ws="
 
 git:
+  # We need a deep clone so that we can compute the age of the files using their git history.
   depth: 10000
 
 install:
-  - export IRIS_TEST_DATA_REF="2f3a6bcf25f81bd152b3d66223394074c9069a96"
-  - export IRIS_TEST_DATA_SUFFIX=$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//")
+  - >
+    export IRIS_TEST_DATA_REF="2f3a6bcf25f81bd152b3d66223394074c9069a96";
+    export IRIS_TEST_DATA_SUFFIX=$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//");
 
   # Install miniconda
   # -----------------
-  - export CONDA_BASE=https://repo.continuum.io/miniconda/Miniconda
-  - if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then
-      wget ${CONDA_BASE}2-latest-Linux-x86_64.sh -O miniconda.sh;
+  - >
+    echo 'Installing miniconda';
+    export CONDA_BASE=https://repo.continuum.io/miniconda/Miniconda;
+    if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then
+      wget --quiet ${CONDA_BASE}2-latest-Linux-x86_64.sh -O miniconda.sh;
     else
-      wget ${CONDA_BASE}3-latest-Linux-x86_64.sh -O miniconda.sh;
-    fi
-  - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$PATH"
+      wget --quiet ${CONDA_BASE}3-latest-Linux-x86_64.sh -O miniconda.sh;
+    fi;
+    bash miniconda.sh -b -p $HOME/miniconda;
+    export PATH="$HOME/miniconda/bin:$PATH";
 
   # Create the basic testing environment
   # ------------------------------------
   # Explicitly add defaults channel, see https://github.com/conda/conda/issues/2675
-  - conda config --add channels defaults
-  - conda config --set always_yes yes --set changeps1 no
-  - conda config --set show_channel_urls True
-  - conda update --quiet conda
-  - ENV_NAME='test-environment'
-  - conda create --quiet -n $ENV_NAME python=$TRAVIS_PYTHON_VERSION
-  - source activate $ENV_NAME
+  - >
+    echo 'Configure conda and create an environment';
+    conda config --set always_yes yes --set changeps1 no;
+    conda config --set show_channel_urls True;
+    conda config --add channels conda-forge;
+    conda update --quiet conda;
+    ENV_NAME='test-environment';
+    conda create --quiet -n $ENV_NAME python=$TRAVIS_PYTHON_VERSION pip;
+    source activate $ENV_NAME;
 
   # Customise the testing environment
   # ---------------------------------
-  - conda config --add channels conda-forge
-  - if [[ "$TEST_MINIMAL" == true ]]; then
-      conda install --quiet --file minimal-conda-requirements.txt;
-    else
-      if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then
-        sed -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet;
-      else
-        conda install --quiet --file conda-requirements.txt;
-      fi
-    fi
-
-  # JUST FOR NOW : Install latest version of iris-grib.
-  # TODO : remove when this release is available on conda-forge.
-  - if [[ "$TEST_MINIMAL" != true ]]; then
-        pip install git+https://github.com/SciTools/[email protected] ;
-    fi
+  - >
+    echo 'Install Iris dependencies';
+    CONDA_REQS_FLAGS="";
+    CONDA_REQS_GROUPS="test";
+    if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then
+        CONDA_REQS_FLAGS="${CONDA_REQS_FLAGS} --py2";
+    fi;
+    if [[ "$TEST_MINIMAL" != true ]]; then
+        CONDA_REQS_GROUPS="${CONDA_REQS_GROUPS} all";
+    fi;
+    if [[ "${TEST_TARGET}" == 'doctest' ]]; then
+        CONDA_REQS_GROUPS="${CONDA_REQS_GROUPS} docs";
+    fi;
+    CONDA_REQS_FILE=conda-requirements.txt;
+    python requirements/gen_conda_requirements.py ${CONDA_REQS_FLAGS} --groups ${CONDA_REQS_GROUPS} > ${CONDA_REQS_FILE};
+    cat ${CONDA_REQS_FILE};
+    conda install --quiet --file ${CONDA_REQS_FILE};
 
   - PREFIX=$HOME/miniconda/envs/$ENV_NAME
 
   # Output debug info
-  - conda list
-  - conda info -a
+  - >
+    conda list;
+    conda info -a;
 
 # Pre-load Natural Earth data to avoid multiple, overlapping downloads.
 # i.e. There should be no DownloadWarning reports in the log.
   - python -c 'import cartopy; cartopy.io.shapereader.natural_earth()'
 
 # iris test data
-  - if [[ "$TEST_MINIMAL" != true ]]; then
-      wget -O iris-test-data.zip https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip;
+  - >
+    if [[ "$TEST_MINIMAL" != true ]]; then
+      wget --quiet -O iris-test-data.zip https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip;
       unzip -q iris-test-data.zip;
-      ln -s $(pwd)/iris-test-data-${IRIS_TEST_DATA_SUFFIX} iris-test-data;
+      mv "iris-test-data-${IRIS_TEST_DATA_SUFFIX}" iris-test-data;
     fi
 
 # set config paths
-  - SITE_CFG=lib/iris/etc/site.cfg
-  - echo "[Resources]" > $SITE_CFG
-  - echo "test_data_dir = $(pwd)/iris-test-data/test_data" >> $SITE_CFG
-  - echo "doc_dir = $(pwd)/docs/iris" >> $SITE_CFG
-  - echo "[System]" >> $SITE_CFG
-  - echo "udunits2_path = $PREFIX/lib/libudunits2.so" >> $SITE_CFG
+  - >
+    SITE_CFG=lib/iris/etc/site.cfg;
+    echo "[Resources]" > $SITE_CFG;
+    echo "test_data_dir = $(pwd)/iris-test-data/test_data" >> $SITE_CFG;
+    echo "doc_dir = $(pwd)/docs/iris" >> $SITE_CFG;
+    echo "[System]" >> $SITE_CFG;
+    echo "udunits2_path = $PREFIX/lib/libudunits2.so" >> $SITE_CFG;
 
   - python setup.py --quiet install
 
+  # JUST FOR NOW : Install latest version of iris-grib.
+  # TODO : remove when iris doesn't do an integration test requiring iris-grib.
+  - if [[ "$TEST_MINIMAL" != true && ${TRAVIS_PYTHON_VERSION} == 2* ]]; then
+        conda install python-ecmwf_grib;
+        pip install git+https://github.com/SciTools/[email protected];
+    fi
+
 script:
   - if [[ $TEST_TARGET == 'default' ]]; then
       python -m iris.tests.runner --default-tests --system-tests --print-failed-images --num-processors=3;

requirements/all.txt

@@ -0,0 +1,17 @@
+# Dependencies for a feature complete installation
+# ------------------------------------------------
+
+# esmpy regridding not available through pip.
+#conda: esmpy>=7.0  (only python=2)
+gdal
+mo_pack
+nc_time_axis
+pandas
+stratify  #conda: python-stratify
+pyugrid
+
+#conda: graphviz
+
+# Iris sample data is not available through pip. It can be installed from
+# https://github.com/SciTools/iris-sample-data/archive/master.zip
+#conda: iris-sample-data

requirements/core.txt

@@ -0,0 +1,12 @@
+# Absolute minimal dependencies for iris
+# --------------------------------------
+
+# Without these, iris won't even import.
+
+cartopy
+matplotlib<1.9
+netcdf4
+numpy
+# pyke (not pip installable)  #conda: pyke
+cf_units
+dask>=0.15.0

requirements/docs.txt

@@ -0,0 +1 @@
+sphinx

requirements/extensions.txt

@@ -0,0 +1,8 @@
+# Iris extensions (i.e. key tools that depend on Iris)
+# ----------------------------------------------------
+
+# Note: pip can handle the circularity of these extensions, but conda will
+# struggle. To install these extensions, ensure iris[core] has been installed
+# first.
+
+iris_grib;python_version<"3"  #conda:

requirements/gen_conda_requirements.py

@@ -0,0 +1,91 @@
+# (C) British Crown Copyright 2017, Met Office
+#
+# This file is part of Iris.
+#
+# Iris is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Iris is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Iris.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import os.path
+
+
+REQS_DIR = os.path.dirname(__file__)
+CONDA_PATTERN = '#conda:'
+
+
+def read_conda_reqs(fname, options):
+    lines = []
+    with open(fname, 'r') as fh:
+        for line in fh:
+            line = line.strip()
+            if '#conda:' in line:
+                line_start = line.index(CONDA_PATTERN) + len(CONDA_PATTERN)
+                line = line[line_start:].strip()
+                if 'only python=2' in line:
+                    if 'python=2' in options:
+                        line = line.replace('(only python=2)', '')
+                        lines.append(line)
+                    else:
+                        continue
+                else:
+                    lines.append(line)
+            else:
+                lines.append(line)
+    return lines
+
+
+def compute_requirements(requirement_names=('core', ), options=None):
+    conda_reqs_lines = []
+
+    if 'python=2' in options:
+        conda_reqs_lines.append('python=2.*')
+    else:
+        conda_reqs_lines.append('# Python 3 conda configuration')
+
+    for req_name in requirement_names:
+        fname = os.path.join(REQS_DIR, '{}.txt'.format(req_name))
+        if not os.path.exists(fname):
+            raise RuntimeError('Unable to find the requirements file for {} '
+                               'in {}'.format(req_name, fname))
+        conda_reqs_lines.extend(read_conda_reqs(fname, options))
+        conda_reqs_lines.append('')
+
+    return conda_reqs_lines
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--verbosity", help="increase output verbosity")
+    parser.add_argument(
+            "--groups", nargs='*', default=[],
+            help=("Gather requirements for these given named groups "
+                  "(as found in the requirements/ folder)"))
+    parser.add_argument(
+            "--py2", action="store_true",
+            help="Build the conda requirements for a python 2 installation")
+
+    args = parser.parse_args()
+
+    requirement_names = args.groups
+    requirement_names.insert(0, 'core')
+    requirement_names.insert(0, 'setup')
+
+    options = []
+    if args.py2:
+        options.append('python=2')
+
+    print('\n'.join(compute_requirements(requirement_names, options)))
+
+
+if __name__ == '__main__':
+    main()

requirements/setup.txt

@@ -0,0 +1,6 @@
+# Dependencies necessary to run setup.py of iris
+# ----------------------------------------------
+
+setuptools
+# pyke (not pip installable)  #conda: pyke
+six

requirements/test.txt

@@ -0,0 +1,9 @@
+# Dependencies needed to run the iris tests
+#------------------------------------------
+
+mock
+nose
+pep8
+filelock
+imagehash
+requests