diff --git a/.travis.yml b/.travis.yml index 134de35..adb8225 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,38 +1,43 @@ +# Based on http://conda.pydata.org/docs/travis.html language: generic -sudo: required + +git: + depth: false + os: -- linux -env: - matrix: - - CONDA_PY=36 + - linux + +branches: + only: + - master + +matrix: + fast_finish: true + include: + - env: CONDA_ENV=build + - env: CONDA_ENV=py36 + - env: CONDA_ENV=py27 + - env: CONDA_ENV=py36-defaults + - env: CONDA_ENV=py37-nodefaults + allow_failures: + - env: CONDA_ENV=py27 + - env: CONDA_ENV=py36-defaults + - env: CONDA_ENV=py37-nodefaults global: secure: cfv2rjBvZZuebRbqRl1zF0INUmK/Uktes4jEbxWHyzUhGYqdGk2+mbZiT+brV85+43Q8xwmce8oiw2ZLA0U/T7L3rVbT6xtlo4blT277DY2kyKcyFIVfkTcUdlu/2i+3O7lqBjzZNXP0lHbNfEQCUAq1k9hz5/gHDsSdkImnwae3EWXma8n7aw3aQlb48McACdjVZdNvvr7lwfIVvcA7QdrgZSwQ3CxlPf5QOMTu2czpwlEXfnxiv0ysv8lrVNrmDObkjLpgVFyxh3yajL126q+hHtPsR4dtDYb615xUpSZd9BU6H27fJdc9nFAGBMxIIlqt9q6q7VJ8mTNPt+2BnPLZtKK+6xvvh7RrKtbwfBKSI0mFWCFSgLMEqka23y9S2jRkrT7Xr0gk32L6AmSItjKXRalVPZHJm4WTLYDYWEOVqpK7xvYVlFFBQ/XWraQUeD7xBz9BPInKm5gugUuPgRqdNA96XEhLX/gEhIE+rZf1AlbtfM7whpV3/pd6d6P+S+YGG3jbfjwJ884JeIovKrM5g4R9E8LAayWTGaSBNSGQX7F3QyZ+g8fIixaKx2JJtFPYE7tt0XbYzbI4Gd1NC/YWGMD+/EDaDUFEpySxGn0FUtbZaYX7czfPQiA44u/CuSpzdBec5LzHyrWmYWWxYxHu75qQQBfqhHAg44ZTR7w= -install: -- | - echo "Installing a fresh version of Miniconda." - MINICONDA_URL="https://repo.continuum.io/miniconda" - MINICONDA_FILE="Miniconda3-latest-$(case $TRAVIS_OS_NAME in (linux) echo Linux;; (osx) echo MacOSX;;esac)-x86_64.sh" - curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}" - bash $MINICONDA_FILE -b -- | - echo "Configuring conda." - source $HOME/miniconda3/bin/activate root - conda install -y conda-build anaconda-client flake8 + +before_install: + - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + +install: bash ci/travis/install.sh + script: -- | - flake8 . - if [ "$TRAVIS_OS_NAME" = "linux" ]; then - conda build -c intake -c defaults -c conda-forge ./conda - else - # Workaround for Travis-CI bug #2: https://github.com/travis-ci/travis-ci/issues/7773 - conda build -c intake -c defaults -c conda-forge --no-test ./conda - fi -- | - if [ -n "$TRAVIS_TAG" ]; then - # If tagged git version, upload package to main channel - anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` - fi -notifications: - email: false - on_success: change - on_failure: always + - which python + - python --version + - if [[ "$CONDA_ENV" == "build" ]]; then + bash ci/travis/build.sh; + else + bash ci/travis/test.sh; + fi diff --git a/README.rst b/README.rst deleted file mode 100644 index 921e600..0000000 --- a/README.rst +++ /dev/null @@ -1,3 +0,0 @@ -intake-xarray: XArray Plugin for Intake ---------------------------------------- - diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..1a8ef61 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,35 @@ +# Based on bokeh appveyor set up +build: false + +platform: + - x64 + +environment: + matrix: + - MINICONDA: C:\Miniconda36-x64 + CONDA_ENV: py36 + - MINICONDA: C:\Miniconda36-x64 + CONDA_ENV: py36-defaults + +matrix: + allow_failures: + - CONDA_ENV: py36-defaults + + +skip_branch_with_pr: true +clone_depth: 5 +skip_tags: true + +init: + - cmd: set PATH=%MINICONDA%;%MINICONDA%\\Scripts;%MINICONDA%\\Library\\bin;%PATH% + - cmd: echo %path% + +install: + - powershell .\\ci\\appveyor\\install.ps1 + - "conda env create -n test_env --file ./ci/environment-%CONDA_ENV%.yml" + - "activate test_env" + - "python setup.py install" + - "conda list" + +test_script: + - "pytest --verbose" diff --git a/ci/appveyor/build.ps1 b/ci/appveyor/build.ps1 new file mode 100644 index 0000000..5254910 --- /dev/null +++ b/ci/appveyor/build.ps1 @@ -0,0 +1,7 @@ +function build(){ + conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask netcdf4 + conda list + conda build -c conda-forge ./conda +} + +build \ No newline at end of file diff --git a/ci/appveyor/install.ps1 b/ci/appveyor/install.ps1 new file mode 100644 index 0000000..cd19c3a --- /dev/null +++ b/ci/appveyor/install.ps1 @@ -0,0 +1,5 @@ +function install() { + conda config --set auto_update_conda off --set always_yes yes --set changeps1 no --set show_channel_urls true +} + +install \ No newline at end of file diff --git a/ci/appveyor/test.ps1 b/ci/appveyor/test.ps1 new file mode 100644 index 0000000..ffe8d47 --- /dev/null +++ b/ci/appveyor/test.ps1 @@ -0,0 +1,9 @@ +function test() { + conda env create -n test_env --file ci/environment-py36.yml + source activate test_env + conda list + pip install --no-deps -e . + pytest --verbose +} + +test \ No newline at end of file diff --git a/ci/environment-py27.yml b/ci/environment-py27.yml new file mode 100644 index 0000000..152ce60 --- /dev/null +++ b/ci/environment-py27.yml @@ -0,0 +1,13 @@ +name: test_env +channels: + - conda-forge +dependencies: + - python=2.7 + - intake>=0.4.1 + - xarray>=0.11.2 + - zarr + - pytest + - netcdf4 + - rasterio + - scikit-image + - pynio diff --git a/ci/environment-py36-defaults.yml b/ci/environment-py36-defaults.yml new file mode 100644 index 0000000..b846cf8 --- /dev/null +++ b/ci/environment-py36-defaults.yml @@ -0,0 +1,10 @@ +name: test_env +dependencies: + - python=3.6 + - intake + - xarray + - zarr + - pytest + - netcdf4 + - rasterio + - scikit-image diff --git a/ci/environment-py36.yml b/ci/environment-py36.yml new file mode 100644 index 0000000..f5bcf3d --- /dev/null +++ b/ci/environment-py36.yml @@ -0,0 +1,12 @@ +name: test_env +channels: + - conda-forge +dependencies: + - python=3.6 + - intake>=0.4.1 + - xarray>=0.11.2 + - zarr + - pytest + - netcdf4 + - rasterio + - scikit-image diff --git a/ci/environment-py37-nodefaults.yml b/ci/environment-py37-nodefaults.yml new file mode 100644 index 0000000..928e740 --- /dev/null +++ b/ci/environment-py37-nodefaults.yml @@ -0,0 +1,13 @@ +name: test_env +channels: + - conda-forge + - nodefaults +dependencies: + - python=3.7 + - intake>=0.4.1 + - xarray=0.11.0 # pinnned this low as a test + - zarr + - pytest + - netcdf4 + - rasterio + - scikit-image diff --git a/ci/travis/build.sh b/ci/travis/build.sh new file mode 100755 index 0000000..69e1ddd --- /dev/null +++ b/ci/travis/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e # exit on error + +echo "Installing dependencies." +conda install -c conda-forge conda-build conda-verify jinja2 intake>=0.4.1 xarray>=0.11.0 zarr dask netcdf4 +conda list + +echo "Building conda package." +conda build -c conda-forge ./conda + +# If tagged, upload package to main channel, otherwise, run tests +if [ -n "$TRAVIS_TAG" ]; then + echo "Uploading conda package." + anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda` +fi diff --git a/ci/travis/install.sh b/ci/travis/install.sh new file mode 100644 index 0000000..26ed4ea --- /dev/null +++ b/ci/travis/install.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e # exit on error + +echo "Configuring conda" +conda config --set auto_update_conda off --set always_yes yes --set changeps1 no --set show_channel_urls true diff --git a/ci/travis/test.sh b/ci/travis/test.sh new file mode 100755 index 0000000..7280f97 --- /dev/null +++ b/ci/travis/test.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e # exit on error + +echo "Creating test env" +conda env create -n test_env --file ci/environment-${CONDA_ENV}.yml +source activate test_env +conda list + +echo "Installing intake_xarray." +pip install --no-deps -e . + +echo "Running tests" +pytest --verbose diff --git a/conda/meta.yaml b/conda/meta.yaml index ac14fca..9a1fbff 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,6 +1,8 @@ +{% set data = load_setup_py_data() %} + package: name: intake-xarray - version: {{ GIT_DESCRIBE_TAG }} + version: {{ data['version'] }} source: path: .. @@ -8,7 +10,7 @@ git: depth: false build: - number: {{ GIT_DESCRIBE_NUMBER }} + number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} script: python setup.py install --single-version-externally-managed --record=record.txt noarch: python @@ -17,11 +19,11 @@ requirements: - python - jinja2 run: - - intake>=0.2 - python - - scipy - - xarray - - zarr + + {% for dep in data['install_requires'] %} + - {{ dep.lower() }} + {% endfor %} test: source_files: @@ -32,8 +34,7 @@ test: - py.test --verbose about: - home: https://github.com/ContinuumIO/intake-xarray - license: BSD + home: {{ data['url'] }} + license: {{ data['license'] }} license_file: LICENSE - summary: | - xarray plugins for Intake + summary: {{ data['description'] }} diff --git a/environment.yml b/environment.yml deleted file mode 100644 index c26812c..0000000 --- a/environment.yml +++ /dev/null @@ -1,162 +0,0 @@ -name: intake-xarray -channels: - - conda-forge - - defaults -dependencies: - - affine=2.2.1=py_0 - - asciitree=0.3.3=py_2 - - autopep8=1.3.4=py_0 - - boost-cpp=1.67.0=h3a22d5f_0 - - boto3=1.7.78=py_0 - - botocore=1.10.78=py_0 - - bzip2=1.0.6=1 - - ca-certificates=2018.4.16=0 - - cairo=1.14.12=he6fea26_5 - - certifi=2018.8.13=py36_0 - - ciocheck=0.1.1=py36_1 - - click-plugins=1.0.3=py_1 - - cligj=0.4.0=py36_0 - - curl=7.61.0=h93b3f91_1 - - docutils=0.14=py36_0 - - expat=2.2.5=hfc679d8_1 - - fasteners=0.14.1=py_3 - - fontconfig=2.13.0=hce039c3_5 - - freetype=2.9.1=h6debe1e_0 - - freexl=1.0.5=hf837533_1 - - geos=3.6.2=hfc679d8_2 - - geotiff=1.4.2=hb54a4aa_2 - - gettext=0.19.8.1=0 - - giflib=5.1.4=h470a237_0 - - glib=2.55.0=h464dc38_2 - - hdf4=4.2.13=0 - - hdf5=1.10.2=hc401514_1 - - icu=58.2=hfc679d8_0 - - jmespath=0.9.3=py_1 - - jpeg=9c=h470a237_0 - - json-c=0.12.1=0 - - kealib=1.4.9=h0bee7d0_2 - - krb5=1.14.6=0 - - libgdal=2.2.4=h25f9fa3_7 - - libiconv=1.15=h470a237_2 - - libkml=1.3.0=hccc92b1_8 - - libnetcdf=4.6.1=h039f2a5_7 - - libpng=1.6.35=ha92aebf_0 - - libpq=9.6.3=0 - - libspatialite=4.3.0a=h3b29d86_21 - - libssh2=1.8.0=h5b517e9_2 - - libtiff=4.0.9=he6b73bb_1 - - libxml2=2.9.8=h422b904_3 - - monotonic=1.5=py_0 - - numcodecs=0.5.5=py36_0 - - openjpeg=2.3.0=h316dc23_3 - - openssl=1.0.2o=h470a237_1 - - pcre=8.41=h470a237_2 - - pixman=0.34.0=2 - - poppler=0.61.1=h4d7e492_4 - - poppler-data=0.4.9=0 - - proj4=4.9.3=5 - - pydocstyle=2.1.1=py36_0 - - pytest-json=0.4.0=py36_0 - - rasterio=1.0.3=py36h1b5fcde_0 - - s3transfer=0.1.13=py36_0 - - snuggs=1.4.1=py_1 - - xerces-c=3.2.0=0 - - yapf=0.21.0=py_0 - - zarr=2.2.0=py_1 - - apipkg=1.5=py36_0 - - appdirs=1.4.3=py36h28b3542_0 - - asn1crypto=0.24.0=py36_0 - - astroid=2.0.4=py36_0 - - atomicwrites=1.1.5=py36_0 - - attrs=18.1.0=py36_0 - - blas=1.0=mkl - - bokeh=0.13.0=py36_0 - - cffi=1.11.5=py36h342bebf_0 - - chardet=3.0.4=py36_1 - - click=6.7=py36hec950be_0 - - cloudpickle=0.5.3=py36_0 - - coverage=4.5.1=py36h1de35cc_0 - - cryptography=2.3=py36hdbc3d79_0 - - cryptography-vectors=2.3=py36_0 - - cytoolz=0.9.0.1=py36h1de35cc_1 - - dask=0.18.2=py36_0 - - dask-core=0.18.2=py36_0 - - distributed=1.22.1=py36_0 - - execnet=1.5.0=py36_0 - - flake8=3.5.0=py36_1 - - heapdict=1.0.0=py36_2 - - idna=2.7=py36_0 - - intel-openmp=2018.0.3=0 - - isort=4.3.4=py36_0 - - jinja2=2.10=py36_0 - - lazy-object-proxy=1.3.1=py36h1de35cc_2 - - libcxx=4.0.1=h579ed51_0 - - libcxxabi=4.0.1=hebd6815_0 - - libedit=3.1.20170329=hb402a30_2 - - libffi=3.2.1=h475c297_4 - - libgfortran=3.0.1=h93005f0_2 - - locket=0.2.0=py36hca03003_1 - - markupsafe=1.0=py36h1de35cc_1 - - mccabe=0.6.1=py36_1 - - mkl=2018.0.3=1 - - mkl_fft=1.0.4=py36h5d10147_1 - - mkl_random=1.0.1=py36h5d10147_1 - - more-itertools=4.3.0=py36_0 - - msgpack-numpy=0.4.3=py36_0 - - msgpack-python=0.5.6=py36h04f5b5a_1 - - ncurses=6.1=h0a44026_0 - - numpy=1.15.0=py36h648b28d_0 - - numpy-base=1.15.0=py36h8a80b8c_0 - - packaging=17.1=py36_0 - - pandas=0.23.4=py36h6440ff4_0 - - partd=0.3.8=py36hf5c4cb8_0 - - pip=10.0.1=py36_0 - - pluggy=0.7.1=py36_0 - - psutil=5.4.6=py36h1de35cc_0 - - py=1.5.4=py36_0 - - pycodestyle=2.3.1=py36h83e8646_0 - - pycparser=2.18=py36_1 - - pyflakes=1.6.0=py36hea45e83_0 - - pylint=2.1.1=py36_0 - - pyopenssl=18.0.0=py36_0 - - pyparsing=2.2.0=py36_1 - - pysocks=1.6.8=py36_0 - - pytest=3.7.1=py36_0 - - pytest-cov=2.5.1=py36_0 - - pytest-forked=0.2=py36_0 - - pytest-xdist=1.22.5=py36_0 - - python=3.6.6=hc167b69_0 - - python-dateutil=2.7.3=py36_0 - - python-snappy=0.5.2=py36h0a44026_0 - - pytz=2018.5=py36_0 - - pyyaml=3.13=py36h1de35cc_0 - - readline=7.0=hc1231fa_4 - - requests=2.19.1=py36_0 - - ruamel_yaml=0.15.46=py36h1de35cc_0 - - scipy=1.1.0=py36hf1f7d93_0 - - setuptools=40.0.0=py36_0 - - six=1.11.0=py36_1 - - snappy=1.1.7=he62c110_3 - - snowballstemmer=1.2.1=py36h6c7b616_0 - - sortedcontainers=2.0.4=py36_0 - - sqlite=3.24.0=ha441bb4_0 - - tblib=1.3.2=py36hda67792_0 - - tk=8.6.7=h35a86e2_3 - - toolz=0.9.0=py36_0 - - tornado=5.1=py36h1de35cc_0 - - typed-ast=1.1.0=py36h1de35cc_0 - - urllib3=1.23=py36_0 - - wheel=0.31.1=py36_0 - - wrapt=1.10.11=py36h1de35cc_2 - - xarray=0.10.8=py36_0 - - xz=5.2.4=h1de35cc_4 - - yaml=0.1.7=hc338f04_2 - - zict=0.1.3=py36_0 - - zlib=1.2.11=hf3cbc9b_2 - - pip: - - holoviews==1.9.5 - - intake==0.2.3 - - msgpack==0.5.6 - - param==1.5.1 -prefix: /Users/mmccarty/anaconda3/envs/intake-xarray - diff --git a/intake_xarray/netcdf.py b/intake_xarray/netcdf.py index 7d52ec2..3786881 100644 --- a/intake_xarray/netcdf.py +++ b/intake_xarray/netcdf.py @@ -1,33 +1,77 @@ # -*- coding: utf-8 -*- -import xarray as xr +from distutils.version import LooseVersion +try: + import xarray as xr + XARRAY_VERSION = LooseVersion(xr.__version__) +except ImportError: + XARRAY_VERSION = None +from intake.source.base import PatternMixin +from intake.source.utils import reverse_format from .base import DataSourceMixin -class NetCDFSource(DataSourceMixin): +class NetCDFSource(DataSourceMixin, PatternMixin): """Open a xarray file. Parameters ---------- - urlpath: str - Path to source file. May include glob "*" characters. Must be a - location in the local file-system. - chunks: int or dict + urlpath : str + Path to source file. May include glob "*" characters, format + pattern strings, or list. + Some examples: + - ``{{ CATALOG_DIR }}/data/air.nc`` + - ``{{ CATALOG_DIR }}/data/*.nc`` + - ``{{ CATALOG_DIR }}/data/air_{year}.nc`` + chunks : int or dict, optional Chunks is used to load the new dataset into dask arrays. ``chunks={}`` loads the dataset with dask using a single chunk for all arrays. + concat_dim : str, optional + Name of dimension along which to concatenate the files. Can + be new or pre-existing. Default is 'concat_dim'. + path_as_pattern : bool or str, optional + Whether to treat the path as a pattern (ie. ``data_{field}.nc``) + and create new coodinates in the output corresponding to pattern + fields. If str, is treated as pattern to match on. Default is True. """ name = 'netcdf' - def __init__(self, urlpath, chunks, xarray_kwargs=None, metadata=None, - **kwargs): + def __init__(self, urlpath, chunks=None, concat_dim='concat_dim', + xarray_kwargs=None, metadata=None, + path_as_pattern=True, **kwargs): + self.path_as_pattern = path_as_pattern self.urlpath = urlpath self.chunks = chunks + self.concat_dim = concat_dim self._kwargs = xarray_kwargs or kwargs self._ds = None super(NetCDFSource, self).__init__(metadata=metadata) def _open_dataset(self): + if not XARRAY_VERSION: + raise ImportError("xarray not available") url = self.urlpath - _open_dataset = xr.open_mfdataset if "*" in url else xr.open_dataset + kwargs = self._kwargs + if "*" in url or isinstance(url, list): + _open_dataset = xr.open_mfdataset + if 'concat_dim' not in kwargs.keys(): + kwargs.update(concat_dim=self.concat_dim) + if self.pattern: + kwargs.update(preprocess=self._add_path_to_ds) + else: + _open_dataset = xr.open_dataset - self._ds = _open_dataset(url, chunks=self.chunks, **self._kwargs) + self._ds = _open_dataset(url, chunks=self.chunks, **kwargs) + + def _add_path_to_ds(self, ds): + """Adding path info to a coord for a particular file + """ + if not (XARRAY_VERSION > '0.11.1'): + raise ImportError("Your version of xarray is '{}'. " + "The insurance that source path is available on output of " + "open_dataset was added in 0.11.2, so " + "pattern urlpaths are not supported.".format(XARRAY_VERSION)) + + var = next(var for var in ds) + new_coords = reverse_format(self.pattern, ds[var].encoding['source']) + return ds.assign_coords(**new_coords) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index dc27a2b..0000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -intake -dask -requests -pandas -scipy -xarray -pytest \ No newline at end of file diff --git a/setup.py b/setup.py index c42788a..0c05b21 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,15 @@ -# -*- coding: utf-8 -*- +#!/usr/bin/env python +#----------------------------------------------------------------------------- +# Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors +# All rights reserved. +# +# The full license is in the LICENSE file, distributed with this software. +#----------------------------------------------------------------------------- from setuptools import setup, find_packages import versioneer -requires = open('requirements.txt').read().strip().split('\n') +INSTALL_REQUIRES = ['intake >=0.4.1', 'xarray >=0.11.0', 'zarr', 'dask', 'netcdf4'] setup( name='intake-xarray', @@ -18,6 +24,7 @@ packages=find_packages(), package_data={'': ['*.csv', '*.yml', '*.html']}, include_package_data=True, - install_requires=requires, - long_description=open('README.rst').read(), + install_requires=INSTALL_REQUIRES, + long_description=open('README.md').read(), + long_description_content_type="text/markdown", zip_safe=False, ) diff --git a/tests/util.py b/tests/conftest.py similarity index 87% rename from tests/util.py rename to tests/conftest.py index b59c906..9d044a9 100644 --- a/tests/util.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -import os +import posixpath import pytest import shutil import tempfile @@ -11,11 +11,11 @@ TEST_DATA_DIR = 'tests/data' TEST_DATA = 'example_1.nc' -TEST_URLPATH = os.path.join(TEST_DATA_DIR, TEST_DATA) +TEST_URLPATH = posixpath.join(TEST_DATA_DIR, TEST_DATA) @pytest.fixture -def cdf_source(): +def netcdf_source(): return NetCDFSource(TEST_URLPATH, {}) diff --git a/tests/data/example_2.nc b/tests/data/example_2.nc new file mode 100644 index 0000000..5775622 Binary files /dev/null and b/tests/data/example_2.nc differ diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 5a76f4a..38b031e 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -4,7 +4,6 @@ import pytest from intake import open_catalog -from .util import dataset # noqa @pytest.fixture diff --git a/tests/test_intake_xarray.py b/tests/test_intake_xarray.py index c8a3473..c189a3f 100644 --- a/tests/test_intake_xarray.py +++ b/tests/test_intake_xarray.py @@ -7,12 +7,10 @@ here = os.path.dirname(__file__) -from .util import TEST_URLPATH, cdf_source, zarr_source, dataset # noqa - -@pytest.mark.parametrize('source', ['cdf', 'zarr']) -def test_discover(source, cdf_source, zarr_source, dataset): - source = {'cdf': cdf_source, 'zarr': zarr_source}[source] +@pytest.mark.parametrize('source', ['netcdf', 'zarr']) +def test_discover(source, netcdf_source, zarr_source, dataset): + source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] r = source.discover() assert r['datashape'] is None @@ -25,9 +23,9 @@ def test_discover(source, cdf_source, zarr_source, dataset): assert set(source.metadata['coords']) == set(dataset.coords.keys()) -@pytest.mark.parametrize('source', ['cdf', 'zarr']) -def test_read(source, cdf_source, zarr_source, dataset): - source = {'cdf': cdf_source, 'zarr': zarr_source}[source] +@pytest.mark.parametrize('source', ['netcdf', 'zarr']) +def test_read(source, netcdf_source, zarr_source, dataset): + source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] ds = source.read_chunked() assert ds.temp.chunks @@ -38,8 +36,8 @@ def test_read(source, cdf_source, zarr_source, dataset): assert np.all(ds.rh == dataset.rh) -def test_read_partition_cdf(cdf_source): - source = cdf_source +def test_read_partition_netcdf(netcdf_source): + source = netcdf_source with pytest.raises(TypeError): source.read_partition(None) out = source.read_partition(('temp', 0, 0, 0, 0)) @@ -48,6 +46,32 @@ def test_read_partition_cdf(cdf_source): assert np.all(out == expected) +def test_read_list_of_netcdf_files(): + from intake_xarray.netcdf import NetCDFSource + source = NetCDFSource([ + os.path.join(here, 'data', 'example_1.nc'), + os.path.join(here, 'data', 'example_2.nc'), + ]) + d = source.to_dask() + assert d.dims == {'lat': 5, 'lon': 10, 'level': 4, 'time': 1, + 'concat_dim': 2} + + +def test_read_glob_pattern_of_netcdf_files(): + """If xarray is old, prompt user to update to use pattern""" + from intake_xarray.netcdf import NetCDFSource, XARRAY_VERSION + source = NetCDFSource(os.path.join(here, 'data', 'example_{num: d}.nc'), + concat_dim='num') + if not (XARRAY_VERSION > '0.11.1'): + with pytest.raises(ImportError, match='open_dataset was added in 0.11.2'): + source.to_dask() + else: + d = source.to_dask() + assert d.dims == {'lat': 5, 'lon': 10, 'level': 4, 'time': 1, + 'num': 2} + assert (d.num.data == np.array([1, 2])).all() + + def test_read_partition_zarr(zarr_source): source = zarr_source with pytest.raises(TypeError): @@ -57,9 +81,9 @@ def test_read_partition_zarr(zarr_source): assert np.all(out == expected) -@pytest.mark.parametrize('source', ['cdf', 'zarr']) -def test_to_dask(source, cdf_source, zarr_source, dataset): - source = {'cdf': cdf_source, 'zarr': zarr_source}[source] +@pytest.mark.parametrize('source', ['netcdf', 'zarr']) +def test_to_dask(source, netcdf_source, zarr_source, dataset): + source = {'netcdf': netcdf_source, 'zarr': zarr_source}[source] ds = source.to_dask() assert ds.dims == dataset.dims @@ -222,14 +246,16 @@ def test_read_pattern_path_as_pattern_as_str_with_list_of_urlpaths(): def test_read_image(): pytest.importorskip('skimage') - im = intake.open_xarray_image(os.path.join(here, 'data', 'little_red.tif')) + from intake_xarray.image import ImageSource + im = ImageSource(os.path.join(here, 'data', 'little_red.tif')) da = im.read() assert da.shape == (64, 64, 3) def test_read_images(): pytest.importorskip('skimage') - im = intake.open_xarray_image(os.path.join(here, 'data', 'little_*.tif')) + from intake_xarray.image import ImageSource + im = ImageSource(os.path.join(here, 'data', 'little_*.tif')) da = im.read() assert da.shape == (2, 64, 64, 3) assert da.dims == ('concat_dim', 'y', 'x', 'channel') @@ -237,8 +263,9 @@ def test_read_images(): def test_read_images_with_pattern(): pytest.importorskip('skimage') + from intake_xarray.image import ImageSource path = os.path.join(here, 'data', 'little_{color}.tif') - im = intake.open_xarray_image(path, concat_dim='color') + im = ImageSource(path, concat_dim='color') da = im.read() assert da.shape == (2, 64, 64, 3) assert len(da.color) == 2 @@ -247,14 +274,16 @@ def test_read_images_with_pattern(): def test_read_images_with_multiple_concat_dims_with_pattern(): pytest.importorskip('skimage') + from intake_xarray.image import ImageSource path = os.path.join(here, 'data', '{size}_{color}.tif') - im = intake.open_xarray_image(path, concat_dim=['size', 'color']) + im = ImageSource(path, concat_dim=['size', 'color']) ds = im.read() assert ds.sel(color='red', size='little').shape == (64, 64, 3) def test_read_jpg_image(): pytest.importorskip('skimage') - im = intake.open_xarray_image(os.path.join(here, 'data', 'dog.jpg')) + from intake_xarray.image import ImageSource + im = ImageSource(os.path.join(here, 'data', 'dog.jpg')) da = im.read() assert da.shape == (192, 192)