From 20685f481026025e0dea5ae03bc2e5eff94691db Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Sun, 18 Nov 2018 20:59:55 -0500 Subject: [PATCH 01/31] Adding path_as_pattern for netcdf --- intake_xarray/netcdf.py | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/intake_xarray/netcdf.py b/intake_xarray/netcdf.py index 7d52ec2..36c64bf 100644 --- a/intake_xarray/netcdf.py +++ b/intake_xarray/netcdf.py @@ -1,25 +1,36 @@ # -*- coding: utf-8 -*- import xarray as xr +from intake.source.base import PatternMixin +from intake.source.utils import reverse_format from .base import DataSourceMixin -class NetCDFSource(DataSourceMixin): +class NetCDFSource(DataSourceMixin, PatternMixin): """Open a xarray file. Parameters ---------- urlpath: str - Path to source file. May include glob "*" characters. Must be a - location in the local file-system. + Path to source file. May include glob "*" characters, format + pattern strings, or list. + Some examples: + - ``{{ CATALOG_DIR }}data/air.nc`` + - ``{{ CATALOG_DIR }}data/*.nc`` + - ``{{ CATALOG_DIR }}data/air_{year}.nc`` chunks: int or dict Chunks is used to load the new dataset into dask arrays. ``chunks={}`` loads the dataset with dask using a single chunk for all arrays. + path_as_pattern: bool or str, optional + Whether to treat the path as a pattern (ie. ``data_{field}.nc``) + and create new coodinates in the output corresponding to pattern + fields. If str, is treated as pattern to match on. Default is True. """ name = 'netcdf' def __init__(self, urlpath, chunks, xarray_kwargs=None, metadata=None, - **kwargs): + path_as_pattern=True, **kwargs): + self.path_as_pattern = path_as_pattern self.urlpath = urlpath self.chunks = chunks self._kwargs = xarray_kwargs or kwargs @@ -28,6 +39,19 @@ def __init__(self, urlpath, chunks, xarray_kwargs=None, metadata=None, def _open_dataset(self): url = self.urlpath - _open_dataset = xr.open_mfdataset if "*" in url else xr.open_dataset + kwargs = self._kwargs + if "*" in url or isinstance(url, list): + _open_dataset = xr.open_mfdataset + if self.pattern: + kwargs.update(preprocess=self._add_path_to_ds) + else: + _open_dataset = xr.open_dataset - self._ds = _open_dataset(url, chunks=self.chunks, **self._kwargs) + self._ds = _open_dataset(url, chunks=self.chunks, **kwargs) + + def _add_path_to_ds(self, ds): + """Adding path info to a coord for a particular file + """ + var = next(var for var in ds) + new_coords = reverse_format(self.pattern, ds[var].encoding['source']) + return ds.assign_coords(**new_coords) From c863bf3a30e90787209d231a5afcb22ff8d60132 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Wed, 23 Jan 2019 11:39:45 -0500 Subject: [PATCH 02/31] Added version checking and new tests --- intake_xarray/netcdf.py | 36 ++++++++++++++++++++------ tests/{util.py => conftest.py} | 6 ++--- tests/data/example_2.nc | Bin 0 -> 1736 bytes tests/test_catalog.py | 1 - tests/test_intake_xarray.py | 46 +++++++++++++++++++++++---------- 5 files changed, 64 insertions(+), 25 deletions(-) rename tests/{util.py => conftest.py} (87%) create mode 100644 tests/data/example_2.nc diff --git a/intake_xarray/netcdf.py b/intake_xarray/netcdf.py index 36c64bf..3786881 100644 --- a/intake_xarray/netcdf.py +++ b/intake_xarray/netcdf.py @@ -1,5 +1,10 @@ # -*- coding: utf-8 -*- -import xarray as xr +from distutils.version import LooseVersion +try: + import xarray as xr + XARRAY_VERSION = LooseVersion(xr.__version__) +except ImportError: + XARRAY_VERSION = None from intake.source.base import PatternMixin from intake.source.utils import reverse_format from .base import DataSourceMixin @@ -10,38 +15,47 @@ class NetCDFSource(DataSourceMixin, PatternMixin): Parameters ---------- - urlpath: str + urlpath : str Path to source file. May include glob "*" characters, format pattern strings, or list. Some examples: - - ``{{ CATALOG_DIR }}data/air.nc`` - - ``{{ CATALOG_DIR }}data/*.nc`` - - ``{{ CATALOG_DIR }}data/air_{year}.nc`` - chunks: int or dict + - ``{{ CATALOG_DIR }}/data/air.nc`` + - ``{{ CATALOG_DIR }}/data/*.nc`` + - ``{{ CATALOG_DIR }}/data/air_{year}.nc`` + chunks : int or dict, optional Chunks is used to load the new dataset into dask arrays. ``chunks={}`` loads the dataset with dask using a single chunk for all arrays. - path_as_pattern: bool or str, optional + concat_dim : str, optional + Name of dimension along which to concatenate the files. Can + be new or pre-existing. Default is 'concat_dim'. + path_as_pattern : bool or str, optional Whether to treat the path as a pattern (ie. ``data_{field}.nc``) and create new coodinates in the output corresponding to pattern fields. If str, is treated as pattern to match on. Default is True. """ name = 'netcdf' - def __init__(self, urlpath, chunks, xarray_kwargs=None, metadata=None, + def __init__(self, urlpath, chunks=None, concat_dim='concat_dim', + xarray_kwargs=None, metadata=None, path_as_pattern=True, **kwargs): self.path_as_pattern = path_as_pattern self.urlpath = urlpath self.chunks = chunks + self.concat_dim = concat_dim self._kwargs = xarray_kwargs or kwargs self._ds = None super(NetCDFSource, self).__init__(metadata=metadata) def _open_dataset(self): + if not XARRAY_VERSION: + raise ImportError("xarray not available") url = self.urlpath kwargs = self._kwargs if "*" in url or isinstance(url, list): _open_dataset = xr.open_mfdataset + if 'concat_dim' not in kwargs.keys(): + kwargs.update(concat_dim=self.concat_dim) if self.pattern: kwargs.update(preprocess=self._add_path_to_ds) else: @@ -52,6 +66,12 @@ def _open_dataset(self): def _add_path_to_ds(self, ds): """Adding path info to a coord for a particular file """ + if not (XARRAY_VERSION > '0.11.1'): + raise ImportError("Your version of xarray is '{}'. " + "The insurance that source path is available on output of " + "open_dataset was added in 0.11.2, so " + "pattern urlpaths are not supported.".format(XARRAY_VERSION)) + var = next(var for var in ds) new_coords = reverse_format(self.pattern, ds[var].encoding['source']) return ds.assign_coords(**new_coords) diff --git a/tests/util.py b/tests/conftest.py similarity index 87% rename from tests/util.py rename to tests/conftest.py index b59c906..9d044a9 100644 --- a/tests/util.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -import os +import posixpath import pytest import shutil import tempfile @@ -11,11 +11,11 @@ TEST_DATA_DIR = 'tests/data' TEST_DATA = 'example_1.nc' -TEST_URLPATH = os.path.join(TEST_DATA_DIR, TEST_DATA) +TEST_URLPATH = posixpath.join(TEST_DATA_DIR, TEST_DATA) @pytest.fixture -def cdf_source(): +def netcdf_source(): return NetCDFSource(TEST_URLPATH, {}) diff --git a/tests/data/example_2.nc b/tests/data/example_2.nc new file mode 100644 index 0000000000000000000000000000000000000000..5775622d0ef85828b436dffcd21366f7538fc55c GIT binary patch literal 1736 zcmeHGF-s#s6dp~Yc*(&DBG?>TUSkmhf^g>+CsvB!AIR+`lVo7BE3>l!!NS_gHda?y z`V%ZIEY`|aYp=Y6D~0%dvl+>W;vdMsmp5IGu&RyscVRC2^#K-J~sbu$S3`%+ZS~^MSIJ z{R3K{1h9^aeB`CSpp&@Uj3eKWXI0io6WPqThLtQsDTW6Szoo4JuK>~gGj4((?oIC&A|Jxw_D*KzjIn%L&8U#czxNE z%WL;?{*P_hHBzR{I5D;u*=e+d7N@8yK@--K$InmulBec*WRy~Q>ih*9=ggh>rmW@c zZ_TsNS6Zu|kr``Do=+&rVf|Ym2Q__*W2uMtkp`)XTQaC`Y^<-=SL@>%@XcLekNb(w z0A>_xz}L}e^aHTK3GfTJ0CM0KxCb79zi9m%*vCwlkmTPH^q*p?!SoHh{suz)xi)w5 zo6Q`c9S+aD&sexJJPR9*#y7?hqn(7yK|Dk);qqV|$JqXt)9G|V;5m=|Ccs{fYn+D> R#B**A-U+x|pI1Acc>;h`|0@6h literal 0 HcmV?d00001 diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 5a76f4a..38b031e 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -4,7 +4,6 @@ import pytest from intake import open_catalog -from .util import dataset # noqa @pytest.fixture diff --git a/tests/test_intake_xarray.py b/tests/test_intake_xarray.py index c8a3473..128c6c1 100644 --- a/tests/test_intake_xarray.py +++ b/tests/test_intake_xarray.py @@ -7,12 +7,10 @@ here = os.path.dirname(__file__) -from .util import TEST_URLPATH, cdf_source, zarr_source, dataset # noqa - -@pytest.mark.parametrize('source', ['cdf', 'zarr']) -def test_discover(source, cdf_source, zarr_source, dataset): - source = {'cdf': cdf_source, 'zarr': zarr_source}[source] +@pytest.mark.parametrize('source', ['netcdf', 'zarr']) +def test_discover(source, netcdf_source, zarr_source, dataset): + source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] r = source.discover() assert r['datashape'] is None @@ -25,9 +23,9 @@ def test_discover(source, cdf_source, zarr_source, dataset): assert set(source.metadata['coords']) == set(dataset.coords.keys()) -@pytest.mark.parametrize('source', ['cdf', 'zarr']) -def test_read(source, cdf_source, zarr_source, dataset): - source = {'cdf': cdf_source, 'zarr': zarr_source}[source] +@pytest.mark.parametrize('source', ['netcdf', 'zarr']) +def test_read(source, netcdf_source, zarr_source, dataset): + source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] ds = source.read_chunked() assert ds.temp.chunks @@ -38,8 +36,8 @@ def test_read(source, cdf_source, zarr_source, dataset): assert np.all(ds.rh == dataset.rh) -def test_read_partition_cdf(cdf_source): - source = cdf_source +def test_read_partition_netcdf(netcdf_source): + source = netcdf_source with pytest.raises(TypeError): source.read_partition(None) out = source.read_partition(('temp', 0, 0, 0, 0)) @@ -48,6 +46,28 @@ def test_read_partition_cdf(cdf_source): assert np.all(out == expected) +def test_read_list_of_netcdf_files(): + from intake_xarray.netcdf import NetCDFSource + source = NetCDFSource([ + os.path.join(here, 'data', 'example_1.nc'), + os.path.join(here, 'data', 'example_2.nc'), + ]) + d = source.to_dask() + assert d.dims == {'lat': 5, 'lon': 10, 'level': 4, 'time': 1, + 'concat_dim': 2} + + +def test_read_glob_pattern_of_netcdf_files(): + from intake_xarray.netcdf import NetCDFSource + + source = NetCDFSource(os.path.join(here, 'data', 'example_{num: d}.nc'), + concat_dim='num') + d = source.to_dask() + assert d.dims == {'lat': 5, 'lon': 10, 'level': 4, 'time': 1, + 'num': 2} + assert (d.num.data == np.array([1, 2])).all() + + def test_read_partition_zarr(zarr_source): source = zarr_source with pytest.raises(TypeError): @@ -57,9 +77,9 @@ def test_read_partition_zarr(zarr_source): assert np.all(out == expected) -@pytest.mark.parametrize('source', ['cdf', 'zarr']) -def test_to_dask(source, cdf_source, zarr_source, dataset): - source = {'cdf': cdf_source, 'zarr': zarr_source}[source] +@pytest.mark.parametrize('source', ['netcdf', 'zarr']) +def test_to_dask(source, netcdf_source, zarr_source, dataset): + source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] ds = source.to_dask() assert ds.dims == dataset.dims From b35ebd8a7d34ab35b1bd057eb1309f4c875e063f Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Wed, 23 Jan 2019 11:59:09 -0500 Subject: [PATCH 03/31] Changed the order of channels to prioritize conda-forge over defaults --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 134de35..400ede3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,10 +22,10 @@ script: - | flake8 . if [ "$TRAVIS_OS_NAME" = "linux" ]; then - conda build -c intake -c defaults -c conda-forge ./conda + conda build -c intake -c conda-forge -c defaults ./conda else # Workaround for Travis-CI bug #2: https://github.com/travis-ci/travis-ci/issues/7773 - conda build -c intake -c defaults -c conda-forge --no-test ./conda + conda build -c intake -c conda-forge -c defaults --no-test ./conda fi - | if [ -n "$TRAVIS_TAG" ]; then From 7a701b7cb66320937dae3902a583d90cbb88e514 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Wed, 23 Jan 2019 12:24:40 -0500 Subject: [PATCH 04/31] Run more tests on CI and get some more packages before running --- .travis.yml | 3 +++ tests/test_intake_xarray.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 400ede3..393dc85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,9 @@ script: if [ -n "$TRAVIS_TAG" ]; then # If tagged git version, upload package to main channel anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` + else + conda install -c conda-forge -c defaults netcdf4 rasterio pynio + pytest --verbose fi notifications: email: false diff --git a/tests/test_intake_xarray.py b/tests/test_intake_xarray.py index 128c6c1..c6d6ad6 100644 --- a/tests/test_intake_xarray.py +++ b/tests/test_intake_xarray.py @@ -58,8 +58,8 @@ def test_read_list_of_netcdf_files(): def test_read_glob_pattern_of_netcdf_files(): + pytest.importorskip('netcdf4') from intake_xarray.netcdf import NetCDFSource - source = NetCDFSource(os.path.join(here, 'data', 'example_{num: d}.nc'), concat_dim='num') d = source.to_dask() From b4be78282f73246625ddf5f0e42e8281e66514ae Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Wed, 23 Jan 2019 13:02:23 -0500 Subject: [PATCH 05/31] Adding missing -y flag to conda command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 393dc85..424459b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ script: # If tagged git version, upload package to main channel anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` else - conda install -c conda-forge -c defaults netcdf4 rasterio pynio + conda install -y -c conda-forge -c defaults netcdf4 rasterio pynio pytest --verbose fi notifications: From 5f4d69bb32d001b41d25e6dbcc39ea83707ec551 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Wed, 23 Jan 2019 13:36:24 -0500 Subject: [PATCH 06/31] Adding pytest to second set of tests --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 424459b..92017e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,8 @@ script: # If tagged git version, upload package to main channel anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` else - conda install -y -c conda-forge -c defaults netcdf4 rasterio pynio + conda install -y -c conda-forge -c defaults --use-local intake + conda install -y -c conda-forge -c defaults netcdf4 rasterio pynio pytest pytest --verbose fi notifications: From 404000500c327e6eb9ff9e67a7b735e3b34448e4 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 14:09:27 -0500 Subject: [PATCH 07/31] Using local intake-xarray --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 92017e7..9480ca8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ script: # If tagged git version, upload package to main channel anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` else - conda install -y -c conda-forge -c defaults --use-local intake + conda install -y -c conda-forge -c defaults --use-local intake-xarray conda install -y -c conda-forge -c defaults netcdf4 rasterio pynio pytest pytest --verbose fi From d4be74a5cdb3022300fbb3d240238263faf3fd12 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 14:46:12 -0500 Subject: [PATCH 08/31] Updating to intake style ci --- .travis.yml | 50 +++++++++++++-------------------- appveyor.yml | 26 +++++++++++++++++ scripts/ci/appveyor/build.ps1 | 8 ++++++ scripts/ci/appveyor/install.ps1 | 11 ++++++++ scripts/ci/build.sh | 21 ++++++++++++++ scripts/ci/install.sh | 27 ++++++++++++++++++ scripts/deps.py | 14 +++++++++ 7 files changed, 126 insertions(+), 31 deletions(-) create mode 100644 appveyor.yml create mode 100644 scripts/ci/appveyor/build.ps1 create mode 100644 scripts/ci/appveyor/install.ps1 create mode 100755 scripts/ci/build.sh create mode 100755 scripts/ci/install.sh create mode 100644 scripts/deps.py diff --git a/.travis.yml b/.travis.yml index 9480ca8..8a12c64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,42 +1,30 @@ language: generic -sudo: required + +git: + depth: false + os: -- linux + - linux + +branches: + only: + - master + env: matrix: - CONDA_PY=36 global: secure: cfv2rjBvZZuebRbqRl1zF0INUmK/Uktes4jEbxWHyzUhGYqdGk2+mbZiT+brV85+43Q8xwmce8oiw2ZLA0U/T7L3rVbT6xtlo4blT277DY2kyKcyFIVfkTcUdlu/2i+3O7lqBjzZNXP0lHbNfEQCUAq1k9hz5/gHDsSdkImnwae3EWXma8n7aw3aQlb48McACdjVZdNvvr7lwfIVvcA7QdrgZSwQ3CxlPf5QOMTu2czpwlEXfnxiv0ysv8lrVNrmDObkjLpgVFyxh3yajL126q+hHtPsR4dtDYb615xUpSZd9BU6H27fJdc9nFAGBMxIIlqt9q6q7VJ8mTNPt+2BnPLZtKK+6xvvh7RrKtbwfBKSI0mFWCFSgLMEqka23y9S2jRkrT7Xr0gk32L6AmSItjKXRalVPZHJm4WTLYDYWEOVqpK7xvYVlFFBQ/XWraQUeD7xBz9BPInKm5gugUuPgRqdNA96XEhLX/gEhIE+rZf1AlbtfM7whpV3/pd6d6P+S+YGG3jbfjwJ884JeIovKrM5g4R9E8LAayWTGaSBNSGQX7F3QyZ+g8fIixaKx2JJtFPYE7tt0XbYzbI4Gd1NC/YWGMD+/EDaDUFEpySxGn0FUtbZaYX7czfPQiA44u/CuSpzdBec5LzHyrWmYWWxYxHu75qQQBfqhHAg44ZTR7w= -install: -- | - echo "Installing a fresh version of Miniconda." - MINICONDA_URL="https://repo.continuum.io/miniconda" - MINICONDA_FILE="Miniconda3-latest-$(case $TRAVIS_OS_NAME in (linux) echo Linux;; (osx) echo MacOSX;;esac)-x86_64.sh" - curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}" - bash $MINICONDA_FILE -b -- | - echo "Configuring conda." - source $HOME/miniconda3/bin/activate root - conda install -y conda-build anaconda-client flake8 -script: -- | - flake8 . - if [ "$TRAVIS_OS_NAME" = "linux" ]; then - conda build -c intake -c conda-forge -c defaults ./conda - else - # Workaround for Travis-CI bug #2: https://github.com/travis-ci/travis-ci/issues/7773 - conda build -c intake -c conda-forge -c defaults --no-test ./conda - fi -- | - if [ -n "$TRAVIS_TAG" ]; then - # If tagged git version, upload package to main channel - anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` - else - conda install -y -c conda-forge -c defaults --use-local intake-xarray - conda install -y -c conda-forge -c defaults netcdf4 rasterio pynio pytest - pytest --verbose - fi + +before_install: + - export PATH="$HOME/miniconda3/bin:$PATH" + +install: scripts/ci/install.sh + +script: scripts/ci/build.sh + notifications: email: false on_success: change on_failure: always + diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..23ddcbc --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,26 @@ +# Based on bokeh appveyor set up +build: false + +platform: + - x64 + +environment: + matrix: + - MINICONDA: C:\Miniconda36-x64 + +skip_branch_with_pr: true +clone_depth: 5 +skip_tags: true + +init: + - cmd: set PATH=%MINICONDA%;%MINICONDA%\\Scripts;%MINICONDA%\\Library\\bin;%PATH% + - cmd: echo %path% + +install: + - powershell .\\scripts\\ci\\appveyor\\install.ps1 + +build_script: + - powershell .\\scripts\\ci\\appveyor\\build.ps1 + +test_script: + - py.test diff --git a/scripts/ci/appveyor/build.ps1 b/scripts/ci/appveyor/build.ps1 new file mode 100644 index 0000000..ed5005e --- /dev/null +++ b/scripts/ci/appveyor/build.ps1 @@ -0,0 +1,8 @@ +function build(){ + conda build --no-test ./conda + conda install --use-local intake-xarray + conda install netcdf4 rasterio pynio pytest scikit-image + conda list +} + +build \ No newline at end of file diff --git a/scripts/ci/appveyor/install.ps1 b/scripts/ci/appveyor/install.ps1 new file mode 100644 index 0000000..c4cc882 --- /dev/null +++ b/scripts/ci/appveyor/install.ps1 @@ -0,0 +1,11 @@ +function install() { + conda config --set auto_update_conda off + conda config --set always_yes yes + conda config --add channels conda-forge + conda config --get channels + conda update -q conda + conda install jinja2 pyyaml pytest conda-build + conda install $(python scripts/deps.py) +} + +install \ No newline at end of file diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh new file mode 100755 index 0000000..ee95c11 --- /dev/null +++ b/scripts/ci/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e # exit on error + +echo "Building conda package." +conda build -c defaults -c conda-forge --no-test ./conda + +# If tagged, upload package to main channel, otherwise, run tests +if [ -n "$TRAVIS_TAG" ]; then + echo "Uploading conda package." + anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` +else + echo "Installing conda package locally." + conda install -y --use-local intake-xarray + + echo "Installing other test dependencies." + conda install -y -c conda-forge -c defaults netcdf4 rasterio pynio pytest scikit-image + + echo "Running unit tests." + py.test +fi diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh new file mode 100755 index 0000000..29acfe5 --- /dev/null +++ b/scripts/ci/install.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e # exit on error + +CONDA_REQS="conda-build=3.17.0 anaconda-client=1.7.2 conda-verify=3.1.1" + +PLATFORM="$(case $TRAVIS_OS_NAME in (linux) echo Linux;; (osx) echo MacOSX;;esac)" + +echo "Installing Miniconda." +MINICONDA_URL="https://repo.continuum.io/miniconda" +MINICONDA_FILENAME=Miniconda3-latest-${PLATFORM}-x86_64.sh +curl -L -O "${MINICONDA_URL}/${MINICONDA_FILENAME}" +bash ${MINICONDA_FILENAME} -b + +# if emergency, temporary package pins are necessary, they can go here +PINNED_PKGS=$(cat < $HOME/miniconda3/conda-meta/pinned + +echo "Configuring conda." +conda config --set auto_update_conda off +conda install --yes ${CONDA_REQS} + +echo "Installing test dependencies." +conda install --yes `python scripts/deps.py` diff --git a/scripts/deps.py b/scripts/deps.py new file mode 100644 index 0000000..d0615fa --- /dev/null +++ b/scripts/deps.py @@ -0,0 +1,14 @@ +import os +import sys + +from conda_build.api import render + +# This is needed because render spams the console +old_stdout = sys.stdout +sys.stdout = open(os.devnull, 'w') +sys.stderr = open(os.devnull, 'w') + +meta = render('conda')[0][0].meta + +sys.stdout = old_stdout +print(' '.join(meta['test']['requires'])) From ecddae6a654de8d8814f78e274062a42b077af26 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 14:51:40 -0500 Subject: [PATCH 09/31] Adding more channel info --- scripts/ci/appveyor/build.ps1 | 6 +++--- scripts/ci/build.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/ci/appveyor/build.ps1 b/scripts/ci/appveyor/build.ps1 index ed5005e..b952fbc 100644 --- a/scripts/ci/appveyor/build.ps1 +++ b/scripts/ci/appveyor/build.ps1 @@ -1,7 +1,7 @@ function build(){ - conda build --no-test ./conda - conda install --use-local intake-xarray - conda install netcdf4 rasterio pynio pytest scikit-image + conda build -c conda-forge -c defaults --no-test ./conda + conda install -c conda-forge -c defaults --use-local intake-xarray + conda install -c conda-forge -c defaults netcdf4 rasterio pynio pytest scikit-image conda list } diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index ee95c11..66f5f28 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -3,7 +3,7 @@ set -e # exit on error echo "Building conda package." -conda build -c defaults -c conda-forge --no-test ./conda +conda build -c conda-forge -c defaults --no-test ./conda # If tagged, upload package to main channel, otherwise, run tests if [ -n "$TRAVIS_TAG" ]; then @@ -11,7 +11,7 @@ if [ -n "$TRAVIS_TAG" ]; then anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` else echo "Installing conda package locally." - conda install -y --use-local intake-xarray + conda install -y -c conda-forge -c defaults --use-local intake-xarray echo "Installing other test dependencies." conda install -y -c conda-forge -c defaults netcdf4 rasterio pynio pytest scikit-image From 1ea46270a3d9c33f24694ef9a43e9db396e6cbf0 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 15:09:35 -0500 Subject: [PATCH 10/31] Cleaning up packaging --- README.rst | 3 - conda/meta.yaml | 15 ++-- environment.yml | 162 ---------------------------------- requirements.txt | 10 +-- scripts/ci/appveyor/build.ps1 | 2 +- setup.py | 14 ++- 6 files changed, 22 insertions(+), 184 deletions(-) delete mode 100644 README.rst delete mode 100644 environment.yml diff --git a/README.rst b/README.rst deleted file mode 100644 index 921e600..0000000 --- a/README.rst +++ /dev/null @@ -1,3 +0,0 @@ -intake-xarray: XArray Plugin for Intake ---------------------------------------- - diff --git a/conda/meta.yaml b/conda/meta.yaml index ac14fca..d7afad7 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,3 +1,5 @@ +{% set data = load_setup_py_data() %} + package: name: intake-xarray version: {{ GIT_DESCRIBE_TAG }} @@ -17,11 +19,9 @@ requirements: - python - jinja2 run: - - intake>=0.2 - python - - scipy - - xarray - - zarr + + {% for dep in data['install_requires'] %} test: source_files: @@ -32,8 +32,7 @@ test: - py.test --verbose about: - home: https://github.com/ContinuumIO/intake-xarray - license: BSD + home: {{ data['url'] }} + license: {{ data['license'] }} license_file: LICENSE - summary: | - xarray plugins for Intake + summary: {{ data['description'] }} diff --git a/environment.yml b/environment.yml deleted file mode 100644 index c26812c..0000000 --- a/environment.yml +++ /dev/null @@ -1,162 +0,0 @@ -name: intake-xarray -channels: - - conda-forge - - defaults -dependencies: - - affine=2.2.1=py_0 - - asciitree=0.3.3=py_2 - - autopep8=1.3.4=py_0 - - boost-cpp=1.67.0=h3a22d5f_0 - - boto3=1.7.78=py_0 - - botocore=1.10.78=py_0 - - bzip2=1.0.6=1 - - ca-certificates=2018.4.16=0 - - cairo=1.14.12=he6fea26_5 - - certifi=2018.8.13=py36_0 - - ciocheck=0.1.1=py36_1 - - click-plugins=1.0.3=py_1 - - cligj=0.4.0=py36_0 - - curl=7.61.0=h93b3f91_1 - - docutils=0.14=py36_0 - - expat=2.2.5=hfc679d8_1 - - fasteners=0.14.1=py_3 - - fontconfig=2.13.0=hce039c3_5 - - freetype=2.9.1=h6debe1e_0 - - freexl=1.0.5=hf837533_1 - - geos=3.6.2=hfc679d8_2 - - geotiff=1.4.2=hb54a4aa_2 - - gettext=0.19.8.1=0 - - giflib=5.1.4=h470a237_0 - - glib=2.55.0=h464dc38_2 - - hdf4=4.2.13=0 - - hdf5=1.10.2=hc401514_1 - - icu=58.2=hfc679d8_0 - - jmespath=0.9.3=py_1 - - jpeg=9c=h470a237_0 - - json-c=0.12.1=0 - - kealib=1.4.9=h0bee7d0_2 - - krb5=1.14.6=0 - - libgdal=2.2.4=h25f9fa3_7 - - libiconv=1.15=h470a237_2 - - libkml=1.3.0=hccc92b1_8 - - libnetcdf=4.6.1=h039f2a5_7 - - libpng=1.6.35=ha92aebf_0 - - libpq=9.6.3=0 - - libspatialite=4.3.0a=h3b29d86_21 - - libssh2=1.8.0=h5b517e9_2 - - libtiff=4.0.9=he6b73bb_1 - - libxml2=2.9.8=h422b904_3 - - monotonic=1.5=py_0 - - numcodecs=0.5.5=py36_0 - - openjpeg=2.3.0=h316dc23_3 - - openssl=1.0.2o=h470a237_1 - - pcre=8.41=h470a237_2 - - pixman=0.34.0=2 - - poppler=0.61.1=h4d7e492_4 - - poppler-data=0.4.9=0 - - proj4=4.9.3=5 - - pydocstyle=2.1.1=py36_0 - - pytest-json=0.4.0=py36_0 - - rasterio=1.0.3=py36h1b5fcde_0 - - s3transfer=0.1.13=py36_0 - - snuggs=1.4.1=py_1 - - xerces-c=3.2.0=0 - - yapf=0.21.0=py_0 - - zarr=2.2.0=py_1 - - apipkg=1.5=py36_0 - - appdirs=1.4.3=py36h28b3542_0 - - asn1crypto=0.24.0=py36_0 - - astroid=2.0.4=py36_0 - - atomicwrites=1.1.5=py36_0 - - attrs=18.1.0=py36_0 - - blas=1.0=mkl - - bokeh=0.13.0=py36_0 - - cffi=1.11.5=py36h342bebf_0 - - chardet=3.0.4=py36_1 - - click=6.7=py36hec950be_0 - - cloudpickle=0.5.3=py36_0 - - coverage=4.5.1=py36h1de35cc_0 - - cryptography=2.3=py36hdbc3d79_0 - - cryptography-vectors=2.3=py36_0 - - cytoolz=0.9.0.1=py36h1de35cc_1 - - dask=0.18.2=py36_0 - - dask-core=0.18.2=py36_0 - - distributed=1.22.1=py36_0 - - execnet=1.5.0=py36_0 - - flake8=3.5.0=py36_1 - - heapdict=1.0.0=py36_2 - - idna=2.7=py36_0 - - intel-openmp=2018.0.3=0 - - isort=4.3.4=py36_0 - - jinja2=2.10=py36_0 - - lazy-object-proxy=1.3.1=py36h1de35cc_2 - - libcxx=4.0.1=h579ed51_0 - - libcxxabi=4.0.1=hebd6815_0 - - libedit=3.1.20170329=hb402a30_2 - - libffi=3.2.1=h475c297_4 - - libgfortran=3.0.1=h93005f0_2 - - locket=0.2.0=py36hca03003_1 - - markupsafe=1.0=py36h1de35cc_1 - - mccabe=0.6.1=py36_1 - - mkl=2018.0.3=1 - - mkl_fft=1.0.4=py36h5d10147_1 - - mkl_random=1.0.1=py36h5d10147_1 - - more-itertools=4.3.0=py36_0 - - msgpack-numpy=0.4.3=py36_0 - - msgpack-python=0.5.6=py36h04f5b5a_1 - - ncurses=6.1=h0a44026_0 - - numpy=1.15.0=py36h648b28d_0 - - numpy-base=1.15.0=py36h8a80b8c_0 - - packaging=17.1=py36_0 - - pandas=0.23.4=py36h6440ff4_0 - - partd=0.3.8=py36hf5c4cb8_0 - - pip=10.0.1=py36_0 - - pluggy=0.7.1=py36_0 - - psutil=5.4.6=py36h1de35cc_0 - - py=1.5.4=py36_0 - - pycodestyle=2.3.1=py36h83e8646_0 - - pycparser=2.18=py36_1 - - pyflakes=1.6.0=py36hea45e83_0 - - pylint=2.1.1=py36_0 - - pyopenssl=18.0.0=py36_0 - - pyparsing=2.2.0=py36_1 - - pysocks=1.6.8=py36_0 - - pytest=3.7.1=py36_0 - - pytest-cov=2.5.1=py36_0 - - pytest-forked=0.2=py36_0 - - pytest-xdist=1.22.5=py36_0 - - python=3.6.6=hc167b69_0 - - python-dateutil=2.7.3=py36_0 - - python-snappy=0.5.2=py36h0a44026_0 - - pytz=2018.5=py36_0 - - pyyaml=3.13=py36h1de35cc_0 - - readline=7.0=hc1231fa_4 - - requests=2.19.1=py36_0 - - ruamel_yaml=0.15.46=py36h1de35cc_0 - - scipy=1.1.0=py36hf1f7d93_0 - - setuptools=40.0.0=py36_0 - - six=1.11.0=py36_1 - - snappy=1.1.7=he62c110_3 - - snowballstemmer=1.2.1=py36h6c7b616_0 - - sortedcontainers=2.0.4=py36_0 - - sqlite=3.24.0=ha441bb4_0 - - tblib=1.3.2=py36hda67792_0 - - tk=8.6.7=h35a86e2_3 - - toolz=0.9.0=py36_0 - - tornado=5.1=py36h1de35cc_0 - - typed-ast=1.1.0=py36h1de35cc_0 - - urllib3=1.23=py36_0 - - wheel=0.31.1=py36_0 - - wrapt=1.10.11=py36h1de35cc_2 - - xarray=0.10.8=py36_0 - - xz=5.2.4=h1de35cc_4 - - yaml=0.1.7=hc338f04_2 - - zict=0.1.3=py36_0 - - zlib=1.2.11=hf3cbc9b_2 - - pip: - - holoviews==1.9.5 - - intake==0.2.3 - - msgpack==0.5.6 - - param==1.5.1 -prefix: /Users/mmccarty/anaconda3/envs/intake-xarray - diff --git a/requirements.txt b/requirements.txt index dc27a2b..46283b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,3 @@ -intake -dask -requests -pandas -scipy -xarray -pytest \ No newline at end of file +# This file is also processed by conda-build, so keep spaces as "pkg >=1.2" +intake >=0.3.0 +xarray >=0.11.2 diff --git a/scripts/ci/appveyor/build.ps1 b/scripts/ci/appveyor/build.ps1 index b952fbc..252dd86 100644 --- a/scripts/ci/appveyor/build.ps1 +++ b/scripts/ci/appveyor/build.ps1 @@ -1,7 +1,7 @@ function build(){ conda build -c conda-forge -c defaults --no-test ./conda conda install -c conda-forge -c defaults --use-local intake-xarray - conda install -c conda-forge -c defaults netcdf4 rasterio pynio pytest scikit-image + conda install -c conda-forge -c defaults netcdf4 rasterio pytest scikit-image conda list } diff --git a/setup.py b/setup.py index c42788a..4b63f69 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,16 @@ -# -*- coding: utf-8 -*- +#!/usr/bin/env python +#----------------------------------------------------------------------------- +# Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors +# All rights reserved. +# +# The full license is in the LICENSE file, distributed with this software. +#----------------------------------------------------------------------------- from setuptools import setup, find_packages import versioneer -requires = open('requirements.txt').read().strip().split('\n') +requires = [line.strip() for line in open( + 'requirements.txt').readlines() if not line.startswith("#")] setup( name='intake-xarray', @@ -19,5 +26,6 @@ package_data={'': ['*.csv', '*.yml', '*.html']}, include_package_data=True, install_requires=requires, - long_description=open('README.rst').read(), + long_description=open('README.md').read(), + long_description_content_type="text/markdown", zip_safe=False, ) From 821d6eb77fe9203527d7b1ce7a8db1af8b431228 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 15:22:13 -0500 Subject: [PATCH 11/31] Fixing jinja template --- conda/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/meta.yaml b/conda/meta.yaml index d7afad7..50b68fe 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -22,6 +22,8 @@ requirements: - python {% for dep in data['install_requires'] %} + - {{ dep.lower() }} + {% endfor %} test: source_files: From 76af5a43d4e9e447e531747e17e14b826d976e55 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 16:13:19 -0500 Subject: [PATCH 12/31] Trying to get same python throughout --- conda/meta.yaml | 4 ++-- scripts/ci/appveyor/build.ps1 | 4 ++-- scripts/ci/build.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conda/meta.yaml b/conda/meta.yaml index 50b68fe..9a1fbff 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -2,7 +2,7 @@ package: name: intake-xarray - version: {{ GIT_DESCRIBE_TAG }} + version: {{ data['version'] }} source: path: .. @@ -10,7 +10,7 @@ git: depth: false build: - number: {{ GIT_DESCRIBE_NUMBER }} + number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} script: python setup.py install --single-version-externally-managed --record=record.txt noarch: python diff --git a/scripts/ci/appveyor/build.ps1 b/scripts/ci/appveyor/build.ps1 index 252dd86..3fd8d21 100644 --- a/scripts/ci/appveyor/build.ps1 +++ b/scripts/ci/appveyor/build.ps1 @@ -1,6 +1,6 @@ function build(){ - conda build -c conda-forge -c defaults --no-test ./conda - conda install -c conda-forge -c defaults --use-local intake-xarray + conda build -c defaults -c conda-forge --no-test ./conda + conda install --use-local intake-xarray conda install -c conda-forge -c defaults netcdf4 rasterio pytest scikit-image conda list } diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 66f5f28..766d1e3 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -11,7 +11,7 @@ if [ -n "$TRAVIS_TAG" ]; then anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` else echo "Installing conda package locally." - conda install -y -c conda-forge -c defaults --use-local intake-xarray + conda install -y --use-local intake-xarray echo "Installing other test dependencies." conda install -y -c conda-forge -c defaults netcdf4 rasterio pynio pytest scikit-image From bc6d3b6b663e6396e523ce67706a296f068e3f58 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 16:36:46 -0500 Subject: [PATCH 13/31] Pinning intake to 0.4.1 and adding zarr as dependency --- requirements.txt | 3 ++- scripts/ci/appveyor/build.ps1 | 4 ++-- scripts/ci/build.sh | 4 ++-- scripts/ci/install.sh | 3 +++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 46283b5..bbddbd0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ # This file is also processed by conda-build, so keep spaces as "pkg >=1.2" -intake >=0.3.0 +intake >=0.4.1 xarray >=0.11.2 +zarr diff --git a/scripts/ci/appveyor/build.ps1 b/scripts/ci/appveyor/build.ps1 index 3fd8d21..dd00dd9 100644 --- a/scripts/ci/appveyor/build.ps1 +++ b/scripts/ci/appveyor/build.ps1 @@ -1,7 +1,7 @@ function build(){ - conda build -c defaults -c conda-forge --no-test ./conda + conda build -c conda-forge -c defaults --no-test ./conda conda install --use-local intake-xarray - conda install -c conda-forge -c defaults netcdf4 rasterio pytest scikit-image + conda install netcdf4 rasterio pytest scikit-image conda list } diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 766d1e3..eeb56d0 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -11,10 +11,10 @@ if [ -n "$TRAVIS_TAG" ]; then anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` else echo "Installing conda package locally." - conda install -y --use-local intake-xarray + conda install --use-local intake-xarray echo "Installing other test dependencies." - conda install -y -c conda-forge -c defaults netcdf4 rasterio pynio pytest scikit-image + conda install netcdf4 rasterio pynio pytest scikit-image echo "Running unit tests." py.test diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh index 29acfe5..e09e7f9 100755 --- a/scripts/ci/install.sh +++ b/scripts/ci/install.sh @@ -21,6 +21,9 @@ echo -e "$PINNED_PKGS" > $HOME/miniconda3/conda-meta/pinned echo "Configuring conda." conda config --set auto_update_conda off +conda config --set always_yes yes +conda config --add channels conda-forge +conda config --get channels conda install --yes ${CONDA_REQS} echo "Installing test dependencies." From 7c45ae9b8b9b86bc54be68257e1d043d39f11b67 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 16:38:16 -0500 Subject: [PATCH 14/31] Also add .travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8a12c64..6265eb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ branches: env: matrix: - - CONDA_PY=36 + - CONDA_PY=37 global: secure: cfv2rjBvZZuebRbqRl1zF0INUmK/Uktes4jEbxWHyzUhGYqdGk2+mbZiT+brV85+43Q8xwmce8oiw2ZLA0U/T7L3rVbT6xtlo4blT277DY2kyKcyFIVfkTcUdlu/2i+3O7lqBjzZNXP0lHbNfEQCUAq1k9hz5/gHDsSdkImnwae3EWXma8n7aw3aQlb48McACdjVZdNvvr7lwfIVvcA7QdrgZSwQ3CxlPf5QOMTu2czpwlEXfnxiv0ysv8lrVNrmDObkjLpgVFyxh3yajL126q+hHtPsR4dtDYb615xUpSZd9BU6H27fJdc9nFAGBMxIIlqt9q6q7VJ8mTNPt+2BnPLZtKK+6xvvh7RrKtbwfBKSI0mFWCFSgLMEqka23y9S2jRkrT7Xr0gk32L6AmSItjKXRalVPZHJm4WTLYDYWEOVqpK7xvYVlFFBQ/XWraQUeD7xBz9BPInKm5gugUuPgRqdNA96XEhLX/gEhIE+rZf1AlbtfM7whpV3/pd6d6P+S+YGG3jbfjwJ884JeIovKrM5g4R9E8LAayWTGaSBNSGQX7F3QyZ+g8fIixaKx2JJtFPYE7tt0XbYzbI4Gd1NC/YWGMD+/EDaDUFEpySxGn0FUtbZaYX7czfPQiA44u/CuSpzdBec5LzHyrWmYWWxYxHu75qQQBfqhHAg44ZTR7w= From 8ca61fc8d607e7a9256f20609fd158bc4ae70833 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 17:23:04 -0500 Subject: [PATCH 15/31] Using conda environment --- .travis.yml | 2 +- scripts/ci/appveyor/build.ps1 | 5 +++-- scripts/ci/appveyor/install.ps1 | 5 ++--- scripts/ci/build.sh | 10 ++++++---- scripts/ci/install.sh | 6 +++--- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6265eb5..8a12c64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ branches: env: matrix: - - CONDA_PY=37 + - CONDA_PY=36 global: secure: cfv2rjBvZZuebRbqRl1zF0INUmK/Uktes4jEbxWHyzUhGYqdGk2+mbZiT+brV85+43Q8xwmce8oiw2ZLA0U/T7L3rVbT6xtlo4blT277DY2kyKcyFIVfkTcUdlu/2i+3O7lqBjzZNXP0lHbNfEQCUAq1k9hz5/gHDsSdkImnwae3EWXma8n7aw3aQlb48McACdjVZdNvvr7lwfIVvcA7QdrgZSwQ3CxlPf5QOMTu2czpwlEXfnxiv0ysv8lrVNrmDObkjLpgVFyxh3yajL126q+hHtPsR4dtDYb615xUpSZd9BU6H27fJdc9nFAGBMxIIlqt9q6q7VJ8mTNPt+2BnPLZtKK+6xvvh7RrKtbwfBKSI0mFWCFSgLMEqka23y9S2jRkrT7Xr0gk32L6AmSItjKXRalVPZHJm4WTLYDYWEOVqpK7xvYVlFFBQ/XWraQUeD7xBz9BPInKm5gugUuPgRqdNA96XEhLX/gEhIE+rZf1AlbtfM7whpV3/pd6d6P+S+YGG3jbfjwJ884JeIovKrM5g4R9E8LAayWTGaSBNSGQX7F3QyZ+g8fIixaKx2JJtFPYE7tt0XbYzbI4Gd1NC/YWGMD+/EDaDUFEpySxGn0FUtbZaYX7czfPQiA44u/CuSpzdBec5LzHyrWmYWWxYxHu75qQQBfqhHAg44ZTR7w= diff --git a/scripts/ci/appveyor/build.ps1 b/scripts/ci/appveyor/build.ps1 index dd00dd9..23cc32e 100644 --- a/scripts/ci/appveyor/build.ps1 +++ b/scripts/ci/appveyor/build.ps1 @@ -1,7 +1,8 @@ function build(){ + conda activate intake + conda list conda build -c conda-forge -c defaults --no-test ./conda - conda install --use-local intake-xarray - conda install netcdf4 rasterio pytest scikit-image + conda install --use-local intake-xarray netcdf4 rasterio pytest scikit-image conda list } diff --git a/scripts/ci/appveyor/install.ps1 b/scripts/ci/appveyor/install.ps1 index c4cc882..95158f4 100644 --- a/scripts/ci/appveyor/install.ps1 +++ b/scripts/ci/appveyor/install.ps1 @@ -3,9 +3,8 @@ function install() { conda config --set always_yes yes conda config --add channels conda-forge conda config --get channels - conda update -q conda - conda install jinja2 pyyaml pytest conda-build - conda install $(python scripts/deps.py) + conda config –set verbosity 3 + conda create -n intake python=3.6 conda-build jinja2 pyyaml pytest $(python scripts/deps.py) } install \ No newline at end of file diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index eeb56d0..007db0f 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -2,6 +2,10 @@ set -e # exit on error +echo "Activating environment" +conda activate intake +conda list + echo "Building conda package." conda build -c conda-forge -c defaults --no-test ./conda @@ -11,10 +15,8 @@ if [ -n "$TRAVIS_TAG" ]; then anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` else echo "Installing conda package locally." - conda install --use-local intake-xarray - - echo "Installing other test dependencies." - conda install netcdf4 rasterio pynio pytest scikit-image + conda install --use-local intake-xarray netcdf4 rasterio pynio pytest scikit-image + conda list echo "Running unit tests." py.test diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh index e09e7f9..f4107ff 100755 --- a/scripts/ci/install.sh +++ b/scripts/ci/install.sh @@ -24,7 +24,7 @@ conda config --set auto_update_conda off conda config --set always_yes yes conda config --add channels conda-forge conda config --get channels -conda install --yes ${CONDA_REQS} +conda config –set verbosity 3 -echo "Installing test dependencies." -conda install --yes `python scripts/deps.py` +echo "Installing dependencies." +conda create -n intake python=3.6 ${CONDA_REQS} `python scripts/deps.py` From a7203abe428b2ad3519a424edbe02733de153d6b Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 17:26:23 -0500 Subject: [PATCH 16/31] Fixing syntax --- scripts/ci/appveyor/install.ps1 | 2 +- scripts/ci/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/appveyor/install.ps1 b/scripts/ci/appveyor/install.ps1 index 95158f4..c452b24 100644 --- a/scripts/ci/appveyor/install.ps1 +++ b/scripts/ci/appveyor/install.ps1 @@ -3,7 +3,7 @@ function install() { conda config --set always_yes yes conda config --add channels conda-forge conda config --get channels - conda config –set verbosity 3 + conda config --set verbosity 3 conda create -n intake python=3.6 conda-build jinja2 pyyaml pytest $(python scripts/deps.py) } diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh index f4107ff..3c58acc 100755 --- a/scripts/ci/install.sh +++ b/scripts/ci/install.sh @@ -24,7 +24,7 @@ conda config --set auto_update_conda off conda config --set always_yes yes conda config --add channels conda-forge conda config --get channels -conda config –set verbosity 3 +conda config --set verbosity 3 echo "Installing dependencies." conda create -n intake python=3.6 ${CONDA_REQS} `python scripts/deps.py` From a3bf45092fa7f276b90bf93c44bfc3c34950150b Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 17:33:49 -0500 Subject: [PATCH 17/31] verbosity was too high --- scripts/ci/appveyor/install.ps1 | 2 +- scripts/ci/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/appveyor/install.ps1 b/scripts/ci/appveyor/install.ps1 index c452b24..5e532ea 100644 --- a/scripts/ci/appveyor/install.ps1 +++ b/scripts/ci/appveyor/install.ps1 @@ -3,7 +3,7 @@ function install() { conda config --set always_yes yes conda config --add channels conda-forge conda config --get channels - conda config --set verbosity 3 + conda config --set verbosity 1 conda create -n intake python=3.6 conda-build jinja2 pyyaml pytest $(python scripts/deps.py) } diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh index 3c58acc..83a9b4a 100755 --- a/scripts/ci/install.sh +++ b/scripts/ci/install.sh @@ -24,7 +24,7 @@ conda config --set auto_update_conda off conda config --set always_yes yes conda config --add channels conda-forge conda config --get channels -conda config --set verbosity 3 +conda config --set verbosity 1 echo "Installing dependencies." conda create -n intake python=3.6 ${CONDA_REQS} `python scripts/deps.py` From dce5cdba8e8d537966d8729e0621076cca3175c9 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Thu, 24 Jan 2019 17:46:19 -0500 Subject: [PATCH 18/31] Just get build reqs first --- scripts/ci/appveyor/install.ps1 | 9 ++++----- scripts/ci/install.sh | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/scripts/ci/appveyor/install.ps1 b/scripts/ci/appveyor/install.ps1 index 5e532ea..cf726ac 100644 --- a/scripts/ci/appveyor/install.ps1 +++ b/scripts/ci/appveyor/install.ps1 @@ -1,10 +1,9 @@ function install() { - conda config --set auto_update_conda off - conda config --set always_yes yes + conda update conda + conda config --set auto_update_conda off --set always_yes yes --set verbosity 1 conda config --add channels conda-forge - conda config --get channels - conda config --set verbosity 1 - conda create -n intake python=3.6 conda-build jinja2 pyyaml pytest $(python scripts/deps.py) + conda install conda-build jinja2 pyyaml + create -n intake python=3.6 $(python scripts/deps.py) } install \ No newline at end of file diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh index 83a9b4a..ca11933 100755 --- a/scripts/ci/install.sh +++ b/scripts/ci/install.sh @@ -20,11 +20,10 @@ mkdir -p $HOME/miniconda3/conda-meta echo -e "$PINNED_PKGS" > $HOME/miniconda3/conda-meta/pinned echo "Configuring conda." -conda config --set auto_update_conda off -conda config --set always_yes yes +conda update conda +conda config --set auto_update_conda off --set always_yes yes --set verbosity 1 conda config --add channels conda-forge -conda config --get channels -conda config --set verbosity 1 +conda install ${CONDA_REQS} echo "Installing dependencies." -conda create -n intake python=3.6 ${CONDA_REQS} `python scripts/deps.py` +conda create -n intake python=3.6 `python scripts/deps.py` From a0a1d886e989fe5f384079a68f1e14ac6e7680b7 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 09:52:22 -0500 Subject: [PATCH 19/31] Restructuring --- .travis.yml | 46 +++++++++++++++++++-------------- appveyor.yml | 8 +++--- ci/appveyor/build.ps1 | 7 +++++ ci/appveyor/install.ps1 | 7 +++++ ci/appveyor/test.ps1 | 9 +++++++ ci/environment-py36.yml | 13 ++++++++++ ci/environment-py37.yml | 13 ++++++++++ ci/travis/build.sh | 15 +++++++++++ ci/travis/install.sh | 6 +++++ ci/travis/test.sh | 13 ++++++++++ requirements.txt | 4 --- scripts/ci/appveyor/build.ps1 | 9 ------- scripts/ci/appveyor/install.ps1 | 9 ------- scripts/ci/build.sh | 23 ----------------- scripts/ci/install.sh | 29 --------------------- scripts/deps.py | 14 ---------- setup.py | 5 ++-- 17 files changed, 116 insertions(+), 114 deletions(-) create mode 100644 ci/appveyor/build.ps1 create mode 100644 ci/appveyor/install.ps1 create mode 100644 ci/appveyor/test.ps1 create mode 100644 ci/environment-py36.yml create mode 100644 ci/environment-py37.yml create mode 100755 ci/travis/build.sh create mode 100644 ci/travis/install.sh create mode 100755 ci/travis/test.sh delete mode 100644 requirements.txt delete mode 100644 scripts/ci/appveyor/build.ps1 delete mode 100644 scripts/ci/appveyor/install.ps1 delete mode 100755 scripts/ci/build.sh delete mode 100755 scripts/ci/install.sh delete mode 100644 scripts/deps.py diff --git a/.travis.yml b/.travis.yml index 8a12c64..0050316 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,30 +1,36 @@ -language: generic +# Based on http://conda.pydata.org/docs/travis.html +language: minimal +sudo: false # use container based build git: depth: false -os: - - linux - branches: only: - - master - -env: - matrix: - - CONDA_PY=36 + - master + +matrix: + fast_finish: true + include: + - env: CONDA_ENV=build + - env: CONDA_ENV=py36 + - env: CONDA_ENV=py37 global: secure: cfv2rjBvZZuebRbqRl1zF0INUmK/Uktes4jEbxWHyzUhGYqdGk2+mbZiT+brV85+43Q8xwmce8oiw2ZLA0U/T7L3rVbT6xtlo4blT277DY2kyKcyFIVfkTcUdlu/2i+3O7lqBjzZNXP0lHbNfEQCUAq1k9hz5/gHDsSdkImnwae3EWXma8n7aw3aQlb48McACdjVZdNvvr7lwfIVvcA7QdrgZSwQ3CxlPf5QOMTu2czpwlEXfnxiv0ysv8lrVNrmDObkjLpgVFyxh3yajL126q+hHtPsR4dtDYb615xUpSZd9BU6H27fJdc9nFAGBMxIIlqt9q6q7VJ8mTNPt+2BnPLZtKK+6xvvh7RrKtbwfBKSI0mFWCFSgLMEqka23y9S2jRkrT7Xr0gk32L6AmSItjKXRalVPZHJm4WTLYDYWEOVqpK7xvYVlFFBQ/XWraQUeD7xBz9BPInKm5gugUuPgRqdNA96XEhLX/gEhIE+rZf1AlbtfM7whpV3/pd6d6P+S+YGG3jbfjwJ884JeIovKrM5g4R9E8LAayWTGaSBNSGQX7F3QyZ+g8fIixaKx2JJtFPYE7tt0XbYzbI4Gd1NC/YWGMD+/EDaDUFEpySxGn0FUtbZaYX7czfPQiA44u/CuSpzdBec5LzHyrWmYWWxYxHu75qQQBfqhHAg44ZTR7w= before_install: - - export PATH="$HOME/miniconda3/bin:$PATH" - -install: scripts/ci/install.sh - -script: scripts/ci/build.sh - -notifications: - email: false - on_success: change - on_failure: always - + - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + +install: ci/travis/install.sh + +script: + - which python + - python --version + - if [[ "$CONDA_ENV" == "build" ]]; then + ci/travis/build.sh; + else + ci/travis/test.sh; + fi diff --git a/appveyor.yml b/appveyor.yml index 23ddcbc..7c2e761 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,6 +7,7 @@ platform: environment: matrix: - MINICONDA: C:\Miniconda36-x64 + CONDA_ENV: py36 skip_branch_with_pr: true clone_depth: 5 @@ -17,10 +18,11 @@ init: - cmd: echo %path% install: - - powershell .\\scripts\\ci\\appveyor\\install.ps1 + - powershell .\\ci\\appveyor\\install.ps1 build_script: - - powershell .\\scripts\\ci\\appveyor\\build.ps1 + - powershell .\\ci\\appveyor\\build.ps1 test_script: - - py.test + - powershell .\\ci\\appveyor\\test.ps1 + diff --git a/ci/appveyor/build.ps1 b/ci/appveyor/build.ps1 new file mode 100644 index 0000000..d896f51 --- /dev/null +++ b/ci/appveyor/build.ps1 @@ -0,0 +1,7 @@ +function build(){ + conda install -c conda-forge conda-build jinja2 intake>=0.11.2 xarray>=0.4.2 zarr + conda list + conda build -c conda-forge ./conda +} + +build \ No newline at end of file diff --git a/ci/appveyor/install.ps1 b/ci/appveyor/install.ps1 new file mode 100644 index 0000000..ba3beb4 --- /dev/null +++ b/ci/appveyor/install.ps1 @@ -0,0 +1,7 @@ +function install() { + conda update --yes conda + conda config --set auto_update_conda off --set always_yes yes --set changeps1 no --set show_channel_urls true + +} + +install \ No newline at end of file diff --git a/ci/appveyor/test.ps1 b/ci/appveyor/test.ps1 new file mode 100644 index 0000000..2662bcc --- /dev/null +++ b/ci/appveyor/test.ps1 @@ -0,0 +1,9 @@ +function test() { + conda env create -n test_env --file ci/environment-${CONDA_ENV}.yml + conda activate test_env + conda list + pip install --no-deps -e . + pytest --verbose +} + +test \ No newline at end of file diff --git a/ci/environment-py36.yml b/ci/environment-py36.yml new file mode 100644 index 0000000..35e4b2b --- /dev/null +++ b/ci/environment-py36.yml @@ -0,0 +1,13 @@ +name: test_env +channels: + - conda-forge +dependencies: + - python=3.6 + - intake>=0.4.1 + - xarray>=0.11.2 + - zarr + - pytest + - netcdf4 + - rasterio + # - pynio turned off because it'll fail on windows + - scikit-image diff --git a/ci/environment-py37.yml b/ci/environment-py37.yml new file mode 100644 index 0000000..f558128 --- /dev/null +++ b/ci/environment-py37.yml @@ -0,0 +1,13 @@ +name: test_env +channels: + - conda-forge +dependencies: + - python=3.7 + - intake>=0.4.1 + - xarray>=0.11.2 + - zarr + - pytest + - netcdf4 + - rasterio + - pynio + - scikit-image diff --git a/ci/travis/build.sh b/ci/travis/build.sh new file mode 100755 index 0000000..b916806 --- /dev/null +++ b/ci/travis/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e # exit on error + +echo "Installing dependencies." +conda install -c conda-forge conda-build jinja2 intake>=0.11.2 xarray>=0.4.2 zarr +conda list + +echo "Building conda package." +conda build -c conda-forge ./conda + +# If tagged, upload package to main channel, otherwise, run tests +if [ -n "$TRAVIS_TAG" ]; then + echo "Uploading conda package." + anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` +fi diff --git a/ci/travis/install.sh b/ci/travis/install.sh new file mode 100644 index 0000000..b55d857 --- /dev/null +++ b/ci/travis/install.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e # exit on error + +echo "Configuring conda" +conda update --yes conda +conda config --set auto_update_conda off --set always_yes yes --set changeps1 no --set show_channel_urls true diff --git a/ci/travis/test.sh b/ci/travis/test.sh new file mode 100755 index 0000000..2a01577 --- /dev/null +++ b/ci/travis/test.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e # exit on error + +echo "Creating test env" +conda env create -n test_env --file ci/environment-${CONDA_ENV}.yml +conda activate test_env +conda list + +echo "Installing intake_xarray." +pip install --no-deps -e . + +echo "Running tests" +pytest --verbose diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index bbddbd0..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -# This file is also processed by conda-build, so keep spaces as "pkg >=1.2" -intake >=0.4.1 -xarray >=0.11.2 -zarr diff --git a/scripts/ci/appveyor/build.ps1 b/scripts/ci/appveyor/build.ps1 deleted file mode 100644 index 23cc32e..0000000 --- a/scripts/ci/appveyor/build.ps1 +++ /dev/null @@ -1,9 +0,0 @@ -function build(){ - conda activate intake - conda list - conda build -c conda-forge -c defaults --no-test ./conda - conda install --use-local intake-xarray netcdf4 rasterio pytest scikit-image - conda list -} - -build \ No newline at end of file diff --git a/scripts/ci/appveyor/install.ps1 b/scripts/ci/appveyor/install.ps1 deleted file mode 100644 index cf726ac..0000000 --- a/scripts/ci/appveyor/install.ps1 +++ /dev/null @@ -1,9 +0,0 @@ -function install() { - conda update conda - conda config --set auto_update_conda off --set always_yes yes --set verbosity 1 - conda config --add channels conda-forge - conda install conda-build jinja2 pyyaml - create -n intake python=3.6 $(python scripts/deps.py) -} - -install \ No newline at end of file diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh deleted file mode 100755 index 007db0f..0000000 --- a/scripts/ci/build.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e # exit on error - -echo "Activating environment" -conda activate intake -conda list - -echo "Building conda package." -conda build -c conda-forge -c defaults --no-test ./conda - -# If tagged, upload package to main channel, otherwise, run tests -if [ -n "$TRAVIS_TAG" ]; then - echo "Uploading conda package." - anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` -else - echo "Installing conda package locally." - conda install --use-local intake-xarray netcdf4 rasterio pynio pytest scikit-image - conda list - - echo "Running unit tests." - py.test -fi diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh deleted file mode 100755 index ca11933..0000000 --- a/scripts/ci/install.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -e # exit on error - -CONDA_REQS="conda-build=3.17.0 anaconda-client=1.7.2 conda-verify=3.1.1" - -PLATFORM="$(case $TRAVIS_OS_NAME in (linux) echo Linux;; (osx) echo MacOSX;;esac)" - -echo "Installing Miniconda." -MINICONDA_URL="https://repo.continuum.io/miniconda" -MINICONDA_FILENAME=Miniconda3-latest-${PLATFORM}-x86_64.sh -curl -L -O "${MINICONDA_URL}/${MINICONDA_FILENAME}" -bash ${MINICONDA_FILENAME} -b - -# if emergency, temporary package pins are necessary, they can go here -PINNED_PKGS=$(cat < $HOME/miniconda3/conda-meta/pinned - -echo "Configuring conda." -conda update conda -conda config --set auto_update_conda off --set always_yes yes --set verbosity 1 -conda config --add channels conda-forge -conda install ${CONDA_REQS} - -echo "Installing dependencies." -conda create -n intake python=3.6 `python scripts/deps.py` diff --git a/scripts/deps.py b/scripts/deps.py deleted file mode 100644 index d0615fa..0000000 --- a/scripts/deps.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -import sys - -from conda_build.api import render - -# This is needed because render spams the console -old_stdout = sys.stdout -sys.stdout = open(os.devnull, 'w') -sys.stderr = open(os.devnull, 'w') - -meta = render('conda')[0][0].meta - -sys.stdout = old_stdout -print(' '.join(meta['test']['requires'])) diff --git a/setup.py b/setup.py index 4b63f69..2f040dd 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,7 @@ from setuptools import setup, find_packages import versioneer -requires = [line.strip() for line in open( - 'requirements.txt').readlines() if not line.startswith("#")] +INSTALL_REQUIRES = ['intake >= 0.4.1', 'xarray >= 0.11.2', 'zarr'] setup( name='intake-xarray', @@ -25,7 +24,7 @@ packages=find_packages(), package_data={'': ['*.csv', '*.yml', '*.html']}, include_package_data=True, - install_requires=requires, + install_requires=INSTALL_REQUIRES, long_description=open('README.md').read(), long_description_content_type="text/markdown", zip_safe=False, ) From ebbc38a7db63a3280f298f2c8c0e6a20207fc407 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 10:00:30 -0500 Subject: [PATCH 20/31] Fixing errors --- .travis.yml | 6 ++++-- ci/appveyor/build.ps1 | 2 +- ci/travis/build.sh | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0050316..78feecc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,12 @@ # Based on http://conda.pydata.org/docs/travis.html -language: minimal -sudo: false # use container based build +language: generic git: depth: false +os: + - linux + branches: only: - master diff --git a/ci/appveyor/build.ps1 b/ci/appveyor/build.ps1 index d896f51..d8f6cb0 100644 --- a/ci/appveyor/build.ps1 +++ b/ci/appveyor/build.ps1 @@ -1,5 +1,5 @@ function build(){ - conda install -c conda-forge conda-build jinja2 intake>=0.11.2 xarray>=0.4.2 zarr + conda install -c conda-forge conda-build jinja2 intake>=0.4.1 xarray>=0.11.2 zarr conda list conda build -c conda-forge ./conda } diff --git a/ci/travis/build.sh b/ci/travis/build.sh index b916806..dbba258 100755 --- a/ci/travis/build.sh +++ b/ci/travis/build.sh @@ -2,7 +2,7 @@ set -e # exit on error echo "Installing dependencies." -conda install -c conda-forge conda-build jinja2 intake>=0.11.2 xarray>=0.4.2 zarr +conda install -c conda-forge conda-build jinja2 intake>=0.4.1 xarray>=0.11.2 zarr conda list echo "Building conda package." From ca7669b870883905dfd98888222b2f8dcbd14269 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 10:13:42 -0500 Subject: [PATCH 21/31] Fixing up spacing --- .travis.yml | 7 +++---- ci/appveyor/install.ps1 | 2 -- ci/appveyor/test.ps1 | 2 +- ci/travis/install.sh | 1 - ci/travis/test.sh | 2 +- setup.py | 2 +- 6 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 78feecc..7f3fc6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,15 +24,14 @@ before_install: - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r -install: ci/travis/install.sh +install: bash ci/travis/install.sh script: - which python - python --version - if [[ "$CONDA_ENV" == "build" ]]; then - ci/travis/build.sh; + bash ci/travis/build.sh; else - ci/travis/test.sh; + bash ci/travis/test.sh; fi diff --git a/ci/appveyor/install.ps1 b/ci/appveyor/install.ps1 index ba3beb4..cd19c3a 100644 --- a/ci/appveyor/install.ps1 +++ b/ci/appveyor/install.ps1 @@ -1,7 +1,5 @@ function install() { - conda update --yes conda conda config --set auto_update_conda off --set always_yes yes --set changeps1 no --set show_channel_urls true - } install \ No newline at end of file diff --git a/ci/appveyor/test.ps1 b/ci/appveyor/test.ps1 index 2662bcc..e13d766 100644 --- a/ci/appveyor/test.ps1 +++ b/ci/appveyor/test.ps1 @@ -1,6 +1,6 @@ function test() { conda env create -n test_env --file ci/environment-${CONDA_ENV}.yml - conda activate test_env + activate test_env conda list pip install --no-deps -e . pytest --verbose diff --git a/ci/travis/install.sh b/ci/travis/install.sh index b55d857..26ed4ea 100644 --- a/ci/travis/install.sh +++ b/ci/travis/install.sh @@ -2,5 +2,4 @@ set -e # exit on error echo "Configuring conda" -conda update --yes conda conda config --set auto_update_conda off --set always_yes yes --set changeps1 no --set show_channel_urls true diff --git a/ci/travis/test.sh b/ci/travis/test.sh index 2a01577..7280f97 100755 --- a/ci/travis/test.sh +++ b/ci/travis/test.sh @@ -3,7 +3,7 @@ set -e # exit on error echo "Creating test env" conda env create -n test_env --file ci/environment-${CONDA_ENV}.yml -conda activate test_env +source activate test_env conda list echo "Installing intake_xarray." diff --git a/setup.py b/setup.py index 2f040dd..8b09d73 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup, find_packages import versioneer -INSTALL_REQUIRES = ['intake >= 0.4.1', 'xarray >= 0.11.2', 'zarr'] +INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.2', 'zarr'] setup( name='intake-xarray', From b5ffc67c89a03ad9b78ec18986540dbd595d4aab Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 10:31:27 -0500 Subject: [PATCH 22/31] Adding dask and skipping more if no scipy --- ci/appveyor/build.ps1 | 2 +- ci/travis/build.sh | 2 +- setup.py | 2 +- tests/test_intake_xarray.py | 21 ++++++++++++++++----- tests/test_remote.py | 1 + 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/ci/appveyor/build.ps1 b/ci/appveyor/build.ps1 index d8f6cb0..cd149e0 100644 --- a/ci/appveyor/build.ps1 +++ b/ci/appveyor/build.ps1 @@ -1,5 +1,5 @@ function build(){ - conda install -c conda-forge conda-build jinja2 intake>=0.4.1 xarray>=0.11.2 zarr + conda install -c conda-forge conda-build jinja2 intake>=0.4.1 xarray>=0.11.2 zarr dask conda list conda build -c conda-forge ./conda } diff --git a/ci/travis/build.sh b/ci/travis/build.sh index dbba258..c5780bf 100755 --- a/ci/travis/build.sh +++ b/ci/travis/build.sh @@ -2,7 +2,7 @@ set -e # exit on error echo "Installing dependencies." -conda install -c conda-forge conda-build jinja2 intake>=0.4.1 xarray>=0.11.2 zarr +conda install -c conda-forge conda-build jinja2 intake>=0.4.1 xarray>=0.11.2 zarr dask conda list echo "Building conda package." diff --git a/setup.py b/setup.py index 8b09d73..ef8a351 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup, find_packages import versioneer -INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.2', 'zarr'] +INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.2', 'zarr', 'dask'] setup( name='intake-xarray', diff --git a/tests/test_intake_xarray.py b/tests/test_intake_xarray.py index c6d6ad6..95783ec 100644 --- a/tests/test_intake_xarray.py +++ b/tests/test_intake_xarray.py @@ -10,6 +10,7 @@ @pytest.mark.parametrize('source', ['netcdf', 'zarr']) def test_discover(source, netcdf_source, zarr_source, dataset): + pytest.importorskip('scipy') source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] r = source.discover() @@ -25,6 +26,7 @@ def test_discover(source, netcdf_source, zarr_source, dataset): @pytest.mark.parametrize('source', ['netcdf', 'zarr']) def test_read(source, netcdf_source, zarr_source, dataset): + pytest.importorskip('scipy') source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] ds = source.read_chunked() @@ -37,6 +39,7 @@ def test_read(source, netcdf_source, zarr_source, dataset): def test_read_partition_netcdf(netcdf_source): + pytest.importorskip('scipy') source = netcdf_source with pytest.raises(TypeError): source.read_partition(None) @@ -47,6 +50,7 @@ def test_read_partition_netcdf(netcdf_source): def test_read_list_of_netcdf_files(): + pytest.importorskip('netcdf4') from intake_xarray.netcdf import NetCDFSource source = NetCDFSource([ os.path.join(here, 'data', 'example_1.nc'), @@ -69,6 +73,7 @@ def test_read_glob_pattern_of_netcdf_files(): def test_read_partition_zarr(zarr_source): + pytest.importorskip('scipy') source = zarr_source with pytest.raises(TypeError): source.read_partition(None) @@ -79,6 +84,7 @@ def test_read_partition_zarr(zarr_source): @pytest.mark.parametrize('source', ['netcdf', 'zarr']) def test_to_dask(source, netcdf_source, zarr_source, dataset): + pytest.importorskip('scipy') source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] ds = source.to_dask() @@ -242,14 +248,16 @@ def test_read_pattern_path_as_pattern_as_str_with_list_of_urlpaths(): def test_read_image(): pytest.importorskip('skimage') - im = intake.open_xarray_image(os.path.join(here, 'data', 'little_red.tif')) + from intake_xarray.image import ImageSource + im = ImageSource(os.path.join(here, 'data', 'little_red.tif')) da = im.read() assert da.shape == (64, 64, 3) def test_read_images(): pytest.importorskip('skimage') - im = intake.open_xarray_image(os.path.join(here, 'data', 'little_*.tif')) + from intake_xarray.image import ImageSource + im = ImageSource(os.path.join(here, 'data', 'little_*.tif')) da = im.read() assert da.shape == (2, 64, 64, 3) assert da.dims == ('concat_dim', 'y', 'x', 'channel') @@ -257,8 +265,9 @@ def test_read_images(): def test_read_images_with_pattern(): pytest.importorskip('skimage') + from intake_xarray.image import ImageSource path = os.path.join(here, 'data', 'little_{color}.tif') - im = intake.open_xarray_image(path, concat_dim='color') + im = ImageSource(path, concat_dim='color') da = im.read() assert da.shape == (2, 64, 64, 3) assert len(da.color) == 2 @@ -267,14 +276,16 @@ def test_read_images_with_pattern(): def test_read_images_with_multiple_concat_dims_with_pattern(): pytest.importorskip('skimage') + from intake_xarray.image import ImageSource path = os.path.join(here, 'data', '{size}_{color}.tif') - im = intake.open_xarray_image(path, concat_dim=['size', 'color']) + im = ImageSource(path, concat_dim=['size', 'color']) ds = im.read() assert ds.sel(color='red', size='little').shape == (64, 64, 3) def test_read_jpg_image(): pytest.importorskip('skimage') - im = intake.open_xarray_image(os.path.join(here, 'data', 'dog.jpg')) + from intake_xarray.image import ImageSource + im = ImageSource(os.path.join(here, 'data', 'dog.jpg')) da = im.read() assert da.shape == (192, 192) diff --git a/tests/test_remote.py b/tests/test_remote.py index f84508e..b536a2e 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -33,6 +33,7 @@ def intake_server(): def test_remote_netcdf(intake_server): + pytest.importorskip('scipy') cat_local = intake.open_catalog(cat_file) cat = intake.open_catalog(intake_server) assert 'xarray_source' in cat From 7632f0fdc013e9b44083ae85ec7d3d7627c1ef12 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 10:51:20 -0500 Subject: [PATCH 23/31] Skipping more if no backend --- ci/appveyor/test.ps1 | 2 +- setup.py | 2 +- tests/conftest.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/appveyor/test.ps1 b/ci/appveyor/test.ps1 index e13d766..7a09e49 100644 --- a/ci/appveyor/test.ps1 +++ b/ci/appveyor/test.ps1 @@ -1,5 +1,5 @@ function test() { - conda env create -n test_env --file ci/environment-${CONDA_ENV}.yml + conda env create -n test_env --file ci/environment-py36.yml activate test_env conda list pip install --no-deps -e . diff --git a/setup.py b/setup.py index ef8a351..b00d5a0 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup, find_packages import versioneer -INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.2', 'zarr', 'dask'] +INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.2', 'zarr', 'dask', 'scipy'] setup( name='intake-xarray', diff --git a/tests/conftest.py b/tests/conftest.py index 9d044a9..393607a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,11 +16,13 @@ @pytest.fixture def netcdf_source(): + pytest.importorskip('scipy') return NetCDFSource(TEST_URLPATH, {}) @pytest.fixture def dataset(): + pytest.importorskip('scipy') return xr.open_dataset(TEST_URLPATH) From 5f4bee4dc63719abbafff7be4da7c8c70068a994 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 11:17:32 -0500 Subject: [PATCH 24/31] Fixing tests and config - adding 2.7, let it fail --- .travis.yml | 3 +++ ci/appveyor/build.ps1 | 2 +- ci/appveyor/test.ps1 | 2 +- ci/environment-py27.yml | 13 +++++++++++++ ci/environment-py36.yml | 1 - ci/environment-py37.yml | 1 - ci/travis/build.sh | 2 +- tests/test_intake_xarray.py | 4 ++-- 8 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 ci/environment-py27.yml diff --git a/.travis.yml b/.travis.yml index 7f3fc6e..1708fcf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,9 @@ matrix: - env: CONDA_ENV=build - env: CONDA_ENV=py36 - env: CONDA_ENV=py37 + - env: CONDA_ENV=py27 + allow_failures: + - env: CONDA_ENV=py27 global: secure: cfv2rjBvZZuebRbqRl1zF0INUmK/Uktes4jEbxWHyzUhGYqdGk2+mbZiT+brV85+43Q8xwmce8oiw2ZLA0U/T7L3rVbT6xtlo4blT277DY2kyKcyFIVfkTcUdlu/2i+3O7lqBjzZNXP0lHbNfEQCUAq1k9hz5/gHDsSdkImnwae3EWXma8n7aw3aQlb48McACdjVZdNvvr7lwfIVvcA7QdrgZSwQ3CxlPf5QOMTu2czpwlEXfnxiv0ysv8lrVNrmDObkjLpgVFyxh3yajL126q+hHtPsR4dtDYb615xUpSZd9BU6H27fJdc9nFAGBMxIIlqt9q6q7VJ8mTNPt+2BnPLZtKK+6xvvh7RrKtbwfBKSI0mFWCFSgLMEqka23y9S2jRkrT7Xr0gk32L6AmSItjKXRalVPZHJm4WTLYDYWEOVqpK7xvYVlFFBQ/XWraQUeD7xBz9BPInKm5gugUuPgRqdNA96XEhLX/gEhIE+rZf1AlbtfM7whpV3/pd6d6P+S+YGG3jbfjwJ884JeIovKrM5g4R9E8LAayWTGaSBNSGQX7F3QyZ+g8fIixaKx2JJtFPYE7tt0XbYzbI4Gd1NC/YWGMD+/EDaDUFEpySxGn0FUtbZaYX7czfPQiA44u/CuSpzdBec5LzHyrWmYWWxYxHu75qQQBfqhHAg44ZTR7w= diff --git a/ci/appveyor/build.ps1 b/ci/appveyor/build.ps1 index cd149e0..89f0646 100644 --- a/ci/appveyor/build.ps1 +++ b/ci/appveyor/build.ps1 @@ -1,5 +1,5 @@ function build(){ - conda install -c conda-forge conda-build jinja2 intake>=0.4.1 xarray>=0.11.2 zarr dask + conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.2 zarr dask conda list conda build -c conda-forge ./conda } diff --git a/ci/appveyor/test.ps1 b/ci/appveyor/test.ps1 index 7a09e49..ffe8d47 100644 --- a/ci/appveyor/test.ps1 +++ b/ci/appveyor/test.ps1 @@ -1,6 +1,6 @@ function test() { conda env create -n test_env --file ci/environment-py36.yml - activate test_env + source activate test_env conda list pip install --no-deps -e . pytest --verbose diff --git a/ci/environment-py27.yml b/ci/environment-py27.yml new file mode 100644 index 0000000..152ce60 --- /dev/null +++ b/ci/environment-py27.yml @@ -0,0 +1,13 @@ +name: test_env +channels: + - conda-forge +dependencies: + - python=2.7 + - intake>=0.4.1 + - xarray>=0.11.2 + - zarr + - pytest + - netcdf4 + - rasterio + - scikit-image + - pynio diff --git a/ci/environment-py36.yml b/ci/environment-py36.yml index 35e4b2b..f5bcf3d 100644 --- a/ci/environment-py36.yml +++ b/ci/environment-py36.yml @@ -9,5 +9,4 @@ dependencies: - pytest - netcdf4 - rasterio - # - pynio turned off because it'll fail on windows - scikit-image diff --git a/ci/environment-py37.yml b/ci/environment-py37.yml index f558128..72678ed 100644 --- a/ci/environment-py37.yml +++ b/ci/environment-py37.yml @@ -9,5 +9,4 @@ dependencies: - pytest - netcdf4 - rasterio - - pynio - scikit-image diff --git a/ci/travis/build.sh b/ci/travis/build.sh index c5780bf..2faccd1 100755 --- a/ci/travis/build.sh +++ b/ci/travis/build.sh @@ -2,7 +2,7 @@ set -e # exit on error echo "Installing dependencies." -conda install -c conda-forge conda-build jinja2 intake>=0.4.1 xarray>=0.11.2 zarr dask +conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.2 zarr dask conda list echo "Building conda package." diff --git a/tests/test_intake_xarray.py b/tests/test_intake_xarray.py index 95783ec..081a473 100644 --- a/tests/test_intake_xarray.py +++ b/tests/test_intake_xarray.py @@ -50,7 +50,7 @@ def test_read_partition_netcdf(netcdf_source): def test_read_list_of_netcdf_files(): - pytest.importorskip('netcdf4') + pytest.importorskip('netCDF4') from intake_xarray.netcdf import NetCDFSource source = NetCDFSource([ os.path.join(here, 'data', 'example_1.nc'), @@ -62,7 +62,7 @@ def test_read_list_of_netcdf_files(): def test_read_glob_pattern_of_netcdf_files(): - pytest.importorskip('netcdf4') + pytest.importorskip('netCDF4') from intake_xarray.netcdf import NetCDFSource source = NetCDFSource(os.path.join(here, 'data', 'example_{num: d}.nc'), concat_dim='num') From 30f3479047233e58ada483430b39edeee8a2dc48 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 11:30:01 -0500 Subject: [PATCH 25/31] Allow lower versions of xarray and test only defaults --- .travis.yml | 1 + ci/appveyor/build.ps1 | 2 +- ci/environment-py36-defaults.yml | 10 ++++++++++ ci/environment-py37.yml | 2 +- ci/travis/build.sh | 2 +- setup.py | 2 +- tests/test_intake_xarray.py | 15 ++++++++++----- 7 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 ci/environment-py36-defaults.yml diff --git a/.travis.yml b/.travis.yml index 1708fcf..0c1eeb7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ matrix: - env: CONDA_ENV=py36 - env: CONDA_ENV=py37 - env: CONDA_ENV=py27 + - env: CONDA_ENV=py36-defaults allow_failures: - env: CONDA_ENV=py27 global: diff --git a/ci/appveyor/build.ps1 b/ci/appveyor/build.ps1 index 89f0646..c9d844d 100644 --- a/ci/appveyor/build.ps1 +++ b/ci/appveyor/build.ps1 @@ -1,5 +1,5 @@ function build(){ - conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.2 zarr dask + conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask conda list conda build -c conda-forge ./conda } diff --git a/ci/environment-py36-defaults.yml b/ci/environment-py36-defaults.yml new file mode 100644 index 0000000..b846cf8 --- /dev/null +++ b/ci/environment-py36-defaults.yml @@ -0,0 +1,10 @@ +name: test_env +dependencies: + - python=3.6 + - intake + - xarray + - zarr + - pytest + - netcdf4 + - rasterio + - scikit-image diff --git a/ci/environment-py37.yml b/ci/environment-py37.yml index 72678ed..a7fe762 100644 --- a/ci/environment-py37.yml +++ b/ci/environment-py37.yml @@ -4,7 +4,7 @@ channels: dependencies: - python=3.7 - intake>=0.4.1 - - xarray>=0.11.2 + - xarray=0.11.0 # pinnned this low as a test - zarr - pytest - netcdf4 diff --git a/ci/travis/build.sh b/ci/travis/build.sh index 2faccd1..1f9e7b1 100755 --- a/ci/travis/build.sh +++ b/ci/travis/build.sh @@ -2,7 +2,7 @@ set -e # exit on error echo "Installing dependencies." -conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.2 zarr dask +conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask conda list echo "Building conda package." diff --git a/setup.py b/setup.py index b00d5a0..626080c 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup, find_packages import versioneer -INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.2', 'zarr', 'dask', 'scipy'] +INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.0', 'zarr', 'dask'] setup( name='intake-xarray', diff --git a/tests/test_intake_xarray.py b/tests/test_intake_xarray.py index 081a473..1efaa81 100644 --- a/tests/test_intake_xarray.py +++ b/tests/test_intake_xarray.py @@ -62,14 +62,19 @@ def test_read_list_of_netcdf_files(): def test_read_glob_pattern_of_netcdf_files(): + """If xarray is old, prompt user to update to use pattern""" pytest.importorskip('netCDF4') - from intake_xarray.netcdf import NetCDFSource + from intake_xarray.netcdf import NetCDFSource, XARRAY_VERSION source = NetCDFSource(os.path.join(here, 'data', 'example_{num: d}.nc'), concat_dim='num') - d = source.to_dask() - assert d.dims == {'lat': 5, 'lon': 10, 'level': 4, 'time': 1, - 'num': 2} - assert (d.num.data == np.array([1, 2])).all() + if not (XARRAY_VERSION > '0.11.1'): + with pytest.raises(ImportError, match='open_dataset was added in 0.11.2'): + source.to_dask() + else: + d = source.to_dask() + assert d.dims == {'lat': 5, 'lon': 10, 'level': 4, 'time': 1, + 'num': 2} + assert (d.num.data == np.array([1, 2])).all() def test_read_partition_zarr(zarr_source): From ba80d46ff791e2ce3a6edff2769de83d5ac23a31 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 11:49:32 -0500 Subject: [PATCH 26/31] Allow defaults to fail, move appveyor content up --- .travis.yml | 1 + appveyor.yml | 12 ++++++++++-- ci/appveyor/build.ps1 | 2 +- ci/travis/build.sh | 2 +- setup.py | 2 +- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0c1eeb7..c1a94ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ matrix: - env: CONDA_ENV=py36-defaults allow_failures: - env: CONDA_ENV=py27 + - env: CONDA_ENV=py36-defaults global: secure: cfv2rjBvZZuebRbqRl1zF0INUmK/Uktes4jEbxWHyzUhGYqdGk2+mbZiT+brV85+43Q8xwmce8oiw2ZLA0U/T7L3rVbT6xtlo4blT277DY2kyKcyFIVfkTcUdlu/2i+3O7lqBjzZNXP0lHbNfEQCUAq1k9hz5/gHDsSdkImnwae3EWXma8n7aw3aQlb48McACdjVZdNvvr7lwfIVvcA7QdrgZSwQ3CxlPf5QOMTu2czpwlEXfnxiv0ysv8lrVNrmDObkjLpgVFyxh3yajL126q+hHtPsR4dtDYb615xUpSZd9BU6H27fJdc9nFAGBMxIIlqt9q6q7VJ8mTNPt+2BnPLZtKK+6xvvh7RrKtbwfBKSI0mFWCFSgLMEqka23y9S2jRkrT7Xr0gk32L6AmSItjKXRalVPZHJm4WTLYDYWEOVqpK7xvYVlFFBQ/XWraQUeD7xBz9BPInKm5gugUuPgRqdNA96XEhLX/gEhIE+rZf1AlbtfM7whpV3/pd6d6P+S+YGG3jbfjwJ884JeIovKrM5g4R9E8LAayWTGaSBNSGQX7F3QyZ+g8fIixaKx2JJtFPYE7tt0XbYzbI4Gd1NC/YWGMD+/EDaDUFEpySxGn0FUtbZaYX7czfPQiA44u/CuSpzdBec5LzHyrWmYWWxYxHu75qQQBfqhHAg44ZTR7w= diff --git a/appveyor.yml b/appveyor.yml index 7c2e761..c416b1c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,6 +8,11 @@ environment: matrix: - MINICONDA: C:\Miniconda36-x64 CONDA_ENV: py36 + - MINICONDA: C:\Miniconda36-x64 + CONDA_ENV: py36-defaults + allow_failures: + - CONDA_ENV: py36-defaults + skip_branch_with_pr: true clone_depth: 5 @@ -19,10 +24,13 @@ init: install: - powershell .\\ci\\appveyor\\install.ps1 + - "conda env create -n test_env --file ci/%CONDA_ENV%.yml" build_script: - powershell .\\ci\\appveyor\\build.ps1 test_script: - - powershell .\\ci\\appveyor\\test.ps1 - + - "activate test_env" + - "pip install --no-deps -e ." + - "conda list" + - "pytest --verbose" diff --git a/ci/appveyor/build.ps1 b/ci/appveyor/build.ps1 index c9d844d..c1352b9 100644 --- a/ci/appveyor/build.ps1 +++ b/ci/appveyor/build.ps1 @@ -1,5 +1,5 @@ function build(){ - conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask + conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask scipy conda list conda build -c conda-forge ./conda } diff --git a/ci/travis/build.sh b/ci/travis/build.sh index 1f9e7b1..f369685 100755 --- a/ci/travis/build.sh +++ b/ci/travis/build.sh @@ -2,7 +2,7 @@ set -e # exit on error echo "Installing dependencies." -conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask +conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask scipy conda list echo "Building conda package." diff --git a/setup.py b/setup.py index 626080c..db28727 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup, find_packages import versioneer -INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.0', 'zarr', 'dask'] +INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.0', 'zarr', 'dask', 'scipy'] setup( name='intake-xarray', From cb0c32c76adfa95bb2a70e4746d2d839a9d92ada Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 11:52:19 -0500 Subject: [PATCH 27/31] Fixing up allow_failures --- appveyor.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index c416b1c..fee6ecf 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,8 +10,10 @@ environment: CONDA_ENV: py36 - MINICONDA: C:\Miniconda36-x64 CONDA_ENV: py36-defaults - allow_failures: - - CONDA_ENV: py36-defaults + +matrix: + allow_failures: + - CONDA_ENV: py36-defaults skip_branch_with_pr: true From c2bfab476d149e0fd198be56ddee23a42f15f86a Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 12:03:13 -0500 Subject: [PATCH 28/31] Fixing again --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index fee6ecf..cf9f2da 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,13 +26,13 @@ init: install: - powershell .\\ci\\appveyor\\install.ps1 - - "conda env create -n test_env --file ci/%CONDA_ENV%.yml" + - "conda env create -n test_env --file ./ci/environment-%CONDA_ENV%.yml" build_script: - powershell .\\ci\\appveyor\\build.ps1 test_script: - "activate test_env" - - "pip install --no-deps -e ." + - "python setup.py install" - "conda list" - "pytest --verbose" From f3e0705f0a2d6609c56f1a5dee7b77ed8b010196 Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 12:12:50 -0500 Subject: [PATCH 29/31] Switching to netcdf4 --- ci/appveyor/build.ps1 | 2 +- ci/travis/build.sh | 2 +- setup.py | 2 +- tests/conftest.py | 2 -- tests/test_intake_xarray.py | 7 ------- tests/test_remote.py | 1 - 6 files changed, 3 insertions(+), 13 deletions(-) diff --git a/ci/appveyor/build.ps1 b/ci/appveyor/build.ps1 index c1352b9..5254910 100644 --- a/ci/appveyor/build.ps1 +++ b/ci/appveyor/build.ps1 @@ -1,5 +1,5 @@ function build(){ - conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask scipy + conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask netcdf4 conda list conda build -c conda-forge ./conda } diff --git a/ci/travis/build.sh b/ci/travis/build.sh index f369685..69e1ddd 100755 --- a/ci/travis/build.sh +++ b/ci/travis/build.sh @@ -2,7 +2,7 @@ set -e # exit on error echo "Installing dependencies." -conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask scipy +conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask netcdf4 conda list echo "Building conda package." diff --git a/setup.py b/setup.py index db28727..0c05b21 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup, find_packages import versioneer -INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.0', 'zarr', 'dask', 'scipy'] +INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.0', 'zarr', 'dask', 'netcdf4'] setup( name='intake-xarray', diff --git a/tests/conftest.py b/tests/conftest.py index 393607a..9d044a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,13 +16,11 @@ @pytest.fixture def netcdf_source(): - pytest.importorskip('scipy') return NetCDFSource(TEST_URLPATH, {}) @pytest.fixture def dataset(): - pytest.importorskip('scipy') return xr.open_dataset(TEST_URLPATH) diff --git a/tests/test_intake_xarray.py b/tests/test_intake_xarray.py index 1efaa81..c189a3f 100644 --- a/tests/test_intake_xarray.py +++ b/tests/test_intake_xarray.py @@ -10,7 +10,6 @@ @pytest.mark.parametrize('source', ['netcdf', 'zarr']) def test_discover(source, netcdf_source, zarr_source, dataset): - pytest.importorskip('scipy') source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] r = source.discover() @@ -26,7 +25,6 @@ def test_discover(source, netcdf_source, zarr_source, dataset): @pytest.mark.parametrize('source', ['netcdf', 'zarr']) def test_read(source, netcdf_source, zarr_source, dataset): - pytest.importorskip('scipy') source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] ds = source.read_chunked() @@ -39,7 +37,6 @@ def test_read(source, netcdf_source, zarr_source, dataset): def test_read_partition_netcdf(netcdf_source): - pytest.importorskip('scipy') source = netcdf_source with pytest.raises(TypeError): source.read_partition(None) @@ -50,7 +47,6 @@ def test_read_partition_netcdf(netcdf_source): def test_read_list_of_netcdf_files(): - pytest.importorskip('netCDF4') from intake_xarray.netcdf import NetCDFSource source = NetCDFSource([ os.path.join(here, 'data', 'example_1.nc'), @@ -63,7 +59,6 @@ def test_read_list_of_netcdf_files(): def test_read_glob_pattern_of_netcdf_files(): """If xarray is old, prompt user to update to use pattern""" - pytest.importorskip('netCDF4') from intake_xarray.netcdf import NetCDFSource, XARRAY_VERSION source = NetCDFSource(os.path.join(here, 'data', 'example_{num: d}.nc'), concat_dim='num') @@ -78,7 +73,6 @@ def test_read_glob_pattern_of_netcdf_files(): def test_read_partition_zarr(zarr_source): - pytest.importorskip('scipy') source = zarr_source with pytest.raises(TypeError): source.read_partition(None) @@ -89,7 +83,6 @@ def test_read_partition_zarr(zarr_source): @pytest.mark.parametrize('source', ['netcdf', 'zarr']) def test_to_dask(source, netcdf_source, zarr_source, dataset): - pytest.importorskip('scipy') source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] ds = source.to_dask() diff --git a/tests/test_remote.py b/tests/test_remote.py index b536a2e..f84508e 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -33,7 +33,6 @@ def intake_server(): def test_remote_netcdf(intake_server): - pytest.importorskip('scipy') cat_local = intake.open_catalog(cat_file) cat = intake.open_catalog(intake_server) assert 'xarray_source' in cat From c38c7f08c49dd0fa4e0fe762c96b888adc85d27e Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 12:13:06 -0500 Subject: [PATCH 30/31] Don't try to build on windows --- appveyor.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index cf9f2da..1a8ef61 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,12 +27,9 @@ init: install: - powershell .\\ci\\appveyor\\install.ps1 - "conda env create -n test_env --file ./ci/environment-%CONDA_ENV%.yml" - -build_script: - - powershell .\\ci\\appveyor\\build.ps1 - -test_script: - "activate test_env" - "python setup.py install" - "conda list" + +test_script: - "pytest --verbose" From 71d3ed730586a78a110fb00a53629f3280f05a5c Mon Sep 17 00:00:00 2001 From: Julia Signell Date: Fri, 25 Jan 2019 12:17:57 -0500 Subject: [PATCH 31/31] Add a nodefaults env - allow it to fail --- .travis.yml | 3 ++- ci/{environment-py37.yml => environment-py37-nodefaults.yml} | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) rename ci/{environment-py37.yml => environment-py37-nodefaults.yml} (92%) diff --git a/.travis.yml b/.travis.yml index c1a94ac..adb8225 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,12 +16,13 @@ matrix: include: - env: CONDA_ENV=build - env: CONDA_ENV=py36 - - env: CONDA_ENV=py37 - env: CONDA_ENV=py27 - env: CONDA_ENV=py36-defaults + - env: CONDA_ENV=py37-nodefaults allow_failures: - env: CONDA_ENV=py27 - env: CONDA_ENV=py36-defaults + - env: CONDA_ENV=py37-nodefaults global: secure: cfv2rjBvZZuebRbqRl1zF0INUmK/Uktes4jEbxWHyzUhGYqdGk2+mbZiT+brV85+43Q8xwmce8oiw2ZLA0U/T7L3rVbT6xtlo4blT277DY2kyKcyFIVfkTcUdlu/2i+3O7lqBjzZNXP0lHbNfEQCUAq1k9hz5/gHDsSdkImnwae3EWXma8n7aw3aQlb48McACdjVZdNvvr7lwfIVvcA7QdrgZSwQ3CxlPf5QOMTu2czpwlEXfnxiv0ysv8lrVNrmDObkjLpgVFyxh3yajL126q+hHtPsR4dtDYb615xUpSZd9BU6H27fJdc9nFAGBMxIIlqt9q6q7VJ8mTNPt+2BnPLZtKK+6xvvh7RrKtbwfBKSI0mFWCFSgLMEqka23y9S2jRkrT7Xr0gk32L6AmSItjKXRalVPZHJm4WTLYDYWEOVqpK7xvYVlFFBQ/XWraQUeD7xBz9BPInKm5gugUuPgRqdNA96XEhLX/gEhIE+rZf1AlbtfM7whpV3/pd6d6P+S+YGG3jbfjwJ884JeIovKrM5g4R9E8LAayWTGaSBNSGQX7F3QyZ+g8fIixaKx2JJtFPYE7tt0XbYzbI4Gd1NC/YWGMD+/EDaDUFEpySxGn0FUtbZaYX7czfPQiA44u/CuSpzdBec5LzHyrWmYWWxYxHu75qQQBfqhHAg44ZTR7w= diff --git a/ci/environment-py37.yml b/ci/environment-py37-nodefaults.yml similarity index 92% rename from ci/environment-py37.yml rename to ci/environment-py37-nodefaults.yml index a7fe762..928e740 100644 --- a/ci/environment-py37.yml +++ b/ci/environment-py37-nodefaults.yml @@ -1,6 +1,7 @@ name: test_env channels: - conda-forge + - nodefaults dependencies: - python=3.7 - intake>=0.4.1