Requirements re-haul (#3812)

* requirements overhaul

* remove gen_conda_requirements

* fixup pip requirements and relocate ci

* fix travis-ci

* fix core spacing

* update readthedocs config

.readthedocs.yml

@@ -4,7 +4,7 @@ build:
     image: latest
 
 conda:
-    environment: ci/requirements/readthedocs.yml
+    environment: requirements/ci/readthedocs.yml
 
 sphinx:
     configuration: docs/iris/src/conf.py

.travis.yml

@@ -4,7 +4,6 @@
 # http://about.travis-ci.org/docs/user/languages/python/#Travis-CI-Uses-Isolated-virtualenvs
 # we will use conda to give us a much faster setup time.
 
-
 language: minimal
 dist: xenial
 
@@ -13,17 +12,17 @@ env:
     # The decryption key for the encrypted .github/deploy_key.scitools-docs.enc.
     - secure: "N9/qBUT5CqfC7KQBDy5mIWZcGNuUJk3e/qmKJpotWYV+zwOI4GghJsRce6nFnlRiwl65l5oBEcvf3+sBvUfbZqh7U0MdHpw2tHhr2FSCmMB3bkvARZblh9M37f4da9G9VmRkqnyBM5G5TImXtoq4dusvNWKvLW0qETciaipq7ws="
   matrix:
-    - PYTHON_VERSION=3.6 TEST_TARGET=default TEST_MINIMAL=true
-    - PYTHON_VERSION=3.6 TEST_TARGET=default TEST_BLACK=true
-    - PYTHON_VERSION=3.6 TEST_TARGET=gallery
-    - PYTHON_VERSION=3.7 TEST_TARGET=default TEST_MINIMAL=true
-    - PYTHON_VERSION=3.7 TEST_TARGET=default TEST_BLACK=true
-    - PYTHON_VERSION=3.7 TEST_TARGET=gallery
-    - PYTHON_VERSION=3.7 TEST_TARGET=doctest PUSH_BUILT_DOCS=true
-    - PYTHON_VERSION=3.7 TEST_TARGET=linkcheck
+    - PYTHON_VERSION='36' TEST_TARGET='default' TEST_MINIMAL=true
+    - PYTHON_VERSION='36' TEST_TARGET='default' TEST_BLACK=true
+    - PYTHON_VERSION='36' TEST_TARGET='gallery'
+    - PYTHON_VERSION='37' TEST_TARGET='default' TEST_MINIMAL=true
+    - PYTHON_VERSION='37' TEST_TARGET='default' TEST_BLACK=true
+    - PYTHON_VERSION='37' TEST_TARGET='gallery'
+    - PYTHON_VERSION='37' TEST_TARGET='doctest' PUSH_BUILT_DOCS=true
+    - PYTHON_VERSION='37' TEST_TARGET='linkcheck'
     # TODO: Dependencies for sphinxcontrib-spelling to be in place before this
     #       spelling code block is enabled
-    #- PYTHON_VERSION=3.7 TEST_TARGET=spelling
+    #- PYTHON_VERSION='37' TEST_TARGET='spelling'
 
 git:
   # We need a deep clone so that we can compute the age of the files using their git history.
@@ -43,36 +42,21 @@ install:
     bash miniconda.sh -b -p ${HOME}/miniconda;
     export PATH="${HOME}/miniconda/bin:${PATH}";
 
-  # Create the basic testing environment
-  # ------------------------------------
+  # Create the testing environment
+  # ------------------------------
   # Explicitly add defaults channel, see https://github.com/conda/conda/issues/2675
   - >
     echo 'Configure conda and create an environment';
     conda config --set always_yes yes --set changeps1 no;
     conda config --set show_channel_urls True;
     conda config --add channels conda-forge;
     conda update --quiet conda;
-    export ENV_NAME='test-environment';
-    conda create --quiet -n ${ENV_NAME} python=${PYTHON_VERSION} pip;
+    export ENV_NAME='iris-dev';
+    ENV_FILE="requirements/ci/py${PYTHON_VERSION}.yml";
+    cat ${ENV_FILE};
+    conda env create --quiet --file=${ENV_FILE};
     source activate ${ENV_NAME};
-
-  # Customise the testing environment
-  # ---------------------------------
-  - >
-    echo 'Install Iris dependencies';
-    CONDA_REQS_GROUPS="test";
-    if [[ "${TEST_MINIMAL}" != true ]]; then
-        CONDA_REQS_GROUPS="${CONDA_REQS_GROUPS} all";
-    fi;
-    if [[ "${TEST_TARGET}" == 'doctest' || "${TEST_TARGET}" == 'linkcheck' ]]; then
-            CONDA_REQS_GROUPS="${CONDA_REQS_GROUPS} docs";
-    fi;
-    CONDA_REQS_FILE="conda-requirements.txt";
-    python requirements/gen_conda_requirements.py --groups ${CONDA_REQS_GROUPS} > ${CONDA_REQS_FILE};
-    cat ${CONDA_REQS_FILE};
-    conda install --quiet -n ${ENV_NAME} --file ${CONDA_REQS_FILE};
-
-  - export PREFIX="${HOME}/miniconda/envs/${ENV_NAME}"
+    export PREFIX="${CONDA_PREFIX}";
 
   # Output debug info
   - >
@@ -176,6 +160,7 @@ script:
   # Results appear at https://scitools-docs.github.io/iris/<<branchname>>/index.html
   - if [[ "${ORG}" == 'SciTools' && "${TRAVIS_EVENT_TYPE}" == 'push' && "${PUSH_BUILT_DOCS}" == 'true' ]]; then
       cd ${INSTALL_DIR};
+      conda install --quiet -n ${ENV_NAME} pip;
       pip install doctr;
       doctr deploy --deploy-repo SciTools-docs/iris --built-docs docs/iris/src/_build/html
         --key-path .github/deploy_key.scitools-docs.enc

pyproject.toml

@@ -28,3 +28,9 @@ exclude = '''
   | um_cf_map.py
 )
 '''
+
+[build-system]
+# Defined by PEP 518
+requires = ["scitools-pyke", "setuptools>=40.8.0", "wheel"]
+# Defined by PEP 517
+build-backend = "setuptools.build_meta"

requirements/all.txt

@@ -1,17 +1,7 @@
-# Dependencies for a feature complete installation
-# ------------------------------------------------
+# Optional dependencies.
 
-# esmpy regridding not available through pip.
-#conda: esmpy>=7.0
-#gdal : under review -- not tested at present
 mo_pack
 nc-time-axis
 pandas
-stratify  #conda: python-stratify
 pyugrid
-
-#conda: graphviz
-
-# Iris sample data is not available through pip. It can be installed from
-# https://github.com/SciTools/iris-sample-data/archive/master.zip
-#conda: iris-sample-data
+stratify

requirements/ci/iris.yml

@@ -0,0 +1 @@
+py37.yml

requirements/ci/py36.yml

@@ -0,0 +1,48 @@
+name: iris-dev
+
+channels:
+  - conda-forge
+
+dependencies:
+  - python=3.6
+
+# Setup dependencies.
+  - setuptools>=40.8.0
+  - pyke
+
+# Core dependencies.
+  - cartopy>=0.18
+  - cf-units>=2
+  - cftime
+  - dask>=2
+  - matplotlib
+  - netcdf4
+  - numpy>=1.14
+  - python-xxhash
+  - scipy
+
+# Optional dependencies.
+  - esmpy>=7.0
+  - graphviz
+  - iris-sample-data
+  - mo_pack
+  - nc-time-axis
+  - pandas
+  - python-stratify
+  - pyugrid
+
+# Test dependencies.
+  - asv
+  - black=19.10b0
+  - filelock
+  - imagehash>=4.0
+  - nose
+  - pillow<7
+  - pre-commit
+  - requests
+
+# Documentation dependencies.
+  - sphinx
+  - sphinx-copybutton
+  - sphinx-gallery
+  - sphinx_rtd_theme

requirements/ci/py37.yml

@@ -0,0 +1,48 @@
+name: iris-dev
+
+channels:
+  - conda-forge
+
+dependencies:
+  - python=3.7
+
+# Setup dependencies.
+  - setuptools>=40.8.0
+  - pyke
+
+# Core dependencies.
+  - cartopy>=0.18
+  - cf-units>=2
+  - cftime
+  - dask>=2
+  - matplotlib
+  - netcdf4
+  - numpy>=1.14
+  - python-xxhash
+  - scipy
+
+# Optional dependencies.
+  - esmpy>=7.0
+  - graphviz
+  - iris-sample-data
+  - mo_pack
+  - nc-time-axis
+  - pandas
+  - python-stratify
+  - pyugrid
+
+# Test dependencies.
+  - asv
+  - black=19.10b0
+  - filelock
+  - imagehash>=4.0
+  - nose
+  - pillow<7
+  - pre-commit
+  - requests
+
+# Documentation dependencies.
+  - sphinx
+  - sphinx-copybutton
+  - sphinx-gallery
+  - sphinx_rtd_theme

requirements/ci/readthedocs.yml

@@ -0,0 +1 @@
+iris.yml

requirements/core.txt

@@ -1,14 +1,11 @@
-# Absolute minimal dependencies for iris
-# --------------------------------------
-
-# Without these, iris won't even import.
+# Core dependencies.
 
 cartopy>=0.18
 cf-units>=2
 cftime
-dask[array]>=2  #conda: dask>=2
+dask[array]>=2
 matplotlib
 netcdf4
 numpy>=1.14
 scipy
-xxhash  #conda: python-xxhash
+xxhash

requirements/docs.txt

@@ -1,4 +1,6 @@
+# Documentation dependencies.
+
 sphinx
-sphinx_rtd_theme
 sphinx-copybutton
 sphinx-gallery
+sphinx_rtd_theme