Skip to content

Commit

Permalink
Bugfix in list_engine (#4811)
Browse files Browse the repository at this point in the history
* fix list_engine

* fix store engine and netcdf4

* reve

* revert changes in guess_engine

* add resister of backend if dependencies aere instralled

* style mypy

* fix import

* use import instead of importlib

* black

* replace ImportError with ModuleNotFoundError

* fix typo

* fix typos

* remove else

* Revert remove imports inside backends functions

* Revert remove imports inside cfgrib

* modify check on imports inside the backends

* remove not used import
  • Loading branch information
aurghs authored Jan 19, 2021
1 parent 2956067 commit 7dbbdca
Show file tree
Hide file tree
Showing 12 changed files with 141 additions and 44 deletions.
20 changes: 18 additions & 2 deletions xarray/backends/cfgrib_.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,23 @@
from ..core import indexing
from ..core.utils import Frozen, FrozenDict, close_on_error
from ..core.variable import Variable
from .common import AbstractDataStore, BackendArray, BackendEntrypoint
from .common import (
BACKEND_ENTRYPOINTS,
AbstractDataStore,
BackendArray,
BackendEntrypoint,
)
from .locks import SerializableLock, ensure_lock
from .store import open_backend_dataset_store

try:
import cfgrib

has_cfgrib = True
except ModuleNotFoundError:
has_cfgrib = False


# FIXME: Add a dedicated lock, even if ecCodes is supposed to be thread-safe
# in most circumstances. See:
# https://confluence.ecmwf.int/display/ECC/Frequently+Asked+Questions
Expand Down Expand Up @@ -38,7 +51,6 @@ class CfGribDataStore(AbstractDataStore):
"""

def __init__(self, filename, lock=None, **backend_kwargs):
import cfgrib

if lock is None:
lock = ECCODES_LOCK
Expand Down Expand Up @@ -129,3 +141,7 @@ def open_backend_dataset_cfgrib(
cfgrib_backend = BackendEntrypoint(
open_dataset=open_backend_dataset_cfgrib, guess_can_open=guess_can_open_cfgrib
)


if has_cfgrib:
BACKEND_ENTRYPOINTS["cfgrib"] = cfgrib_backend
4 changes: 4 additions & 0 deletions xarray/backends/common.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
import time
import traceback
from typing import Dict

import numpy as np

Expand Down Expand Up @@ -349,3 +350,6 @@ def __init__(self, open_dataset, open_dataset_parameters=None, guess_can_open=No
self.open_dataset = open_dataset
self.open_dataset_parameters = open_dataset_parameters
self.guess_can_open = guess_can_open


BACKEND_ENTRYPOINTS: Dict[str, BackendEntrypoint] = {}
20 changes: 16 additions & 4 deletions xarray/backends/h5netcdf_.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@
from ..core import indexing
from ..core.utils import FrozenDict, is_remote_uri, read_magic_number
from ..core.variable import Variable
from .common import BackendEntrypoint, WritableCFDataStore, find_root_and_group
from .common import (
BACKEND_ENTRYPOINTS,
BackendEntrypoint,
WritableCFDataStore,
find_root_and_group,
)
from .file_manager import CachingFileManager, DummyFileManager
from .locks import HDF5_LOCK, combine_locks, ensure_lock, get_write_lock
from .netCDF4_ import (
Expand All @@ -20,6 +25,13 @@
)
from .store import open_backend_dataset_store

try:
import h5netcdf

has_h5netcdf = True
except ModuleNotFoundError:
has_h5netcdf = False


class H5NetCDFArrayWrapper(BaseNetCDF4Array):
def get_array(self, needs_lock=True):
Expand Down Expand Up @@ -85,8 +97,6 @@ class H5NetCDFStore(WritableCFDataStore):

def __init__(self, manager, group=None, mode=None, lock=HDF5_LOCK, autoclose=False):

import h5netcdf

if isinstance(manager, (h5netcdf.File, h5netcdf.Group)):
if group is None:
root, group = find_root_and_group(manager)
Expand Down Expand Up @@ -122,7 +132,6 @@ def open(
invalid_netcdf=None,
phony_dims=None,
):
import h5netcdf

if isinstance(filename, bytes):
raise ValueError(
Expand Down Expand Up @@ -375,3 +384,6 @@ def open_backend_dataset_h5netcdf(
h5netcdf_backend = BackendEntrypoint(
open_dataset=open_backend_dataset_h5netcdf, guess_can_open=guess_can_open_h5netcdf
)

if has_h5netcdf:
BACKEND_ENTRYPOINTS["h5netcdf"] = h5netcdf_backend
15 changes: 13 additions & 2 deletions xarray/backends/netCDF4_.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from ..core.utils import FrozenDict, close_on_error, is_remote_uri
from ..core.variable import Variable
from .common import (
BACKEND_ENTRYPOINTS,
BackendArray,
BackendEntrypoint,
WritableCFDataStore,
Expand All @@ -23,6 +24,14 @@
from .netcdf3 import encode_nc3_attr_value, encode_nc3_variable
from .store import open_backend_dataset_store

try:
import netCDF4

has_netcdf4 = True
except ModuleNotFoundError:
has_netcdf4 = False


# This lookup table maps from dtype.byteorder to a readable endian
# string used by netCDF4.
_endian_lookup = {"=": "native", ">": "big", "<": "little", "|": "native"}
Expand Down Expand Up @@ -298,7 +307,6 @@ class NetCDF4DataStore(WritableCFDataStore):
def __init__(
self, manager, group=None, mode=None, lock=NETCDF4_PYTHON_LOCK, autoclose=False
):
import netCDF4

if isinstance(manager, netCDF4.Dataset):
if group is None:
Expand Down Expand Up @@ -335,7 +343,6 @@ def open(
lock_maker=None,
autoclose=False,
):
import netCDF4

if isinstance(filename, pathlib.Path):
filename = os.fspath(filename)
Expand Down Expand Up @@ -563,3 +570,7 @@ def open_backend_dataset_netcdf4(
netcdf4_backend = BackendEntrypoint(
open_dataset=open_backend_dataset_netcdf4, guess_can_open=guess_can_open_netcdf4
)


if has_netcdf4:
BACKEND_ENTRYPOINTS["netcdf4"] = netcdf4_backend
24 changes: 1 addition & 23 deletions xarray/backends/plugins.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,33 +2,11 @@
import inspect
import itertools
import logging
import typing as T
import warnings

import pkg_resources

from .cfgrib_ import cfgrib_backend
from .common import BackendEntrypoint
from .h5netcdf_ import h5netcdf_backend
from .netCDF4_ import netcdf4_backend
from .pseudonetcdf_ import pseudonetcdf_backend
from .pydap_ import pydap_backend
from .pynio_ import pynio_backend
from .scipy_ import scipy_backend
from .store import store_backend
from .zarr import zarr_backend

BACKEND_ENTRYPOINTS: T.Dict[str, BackendEntrypoint] = {
"store": store_backend,
"netcdf4": netcdf4_backend,
"h5netcdf": h5netcdf_backend,
"scipy": scipy_backend,
"pseudonetcdf": pseudonetcdf_backend,
"zarr": zarr_backend,
"cfgrib": cfgrib_backend,
"pydap": pydap_backend,
"pynio": pynio_backend,
}
from .common import BACKEND_ENTRYPOINTS


def remove_duplicates(backend_entrypoints):
Expand Down
20 changes: 18 additions & 2 deletions xarray/backends/pseudonetcdf_.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,24 @@
from ..core import indexing
from ..core.utils import Frozen, FrozenDict, close_on_error
from ..core.variable import Variable
from .common import AbstractDataStore, BackendArray, BackendEntrypoint
from .common import (
BACKEND_ENTRYPOINTS,
AbstractDataStore,
BackendArray,
BackendEntrypoint,
)
from .file_manager import CachingFileManager
from .locks import HDF5_LOCK, NETCDFC_LOCK, combine_locks, ensure_lock
from .store import open_backend_dataset_store

try:
from PseudoNetCDF import pncopen

has_pseudonetcdf = True
except ModuleNotFoundError:
has_pseudonetcdf = False


# psuedonetcdf can invoke netCDF libraries internally
PNETCDF_LOCK = combine_locks([HDF5_LOCK, NETCDFC_LOCK])

Expand Down Expand Up @@ -40,7 +53,6 @@ class PseudoNetCDFDataStore(AbstractDataStore):

@classmethod
def open(cls, filename, lock=None, mode=None, **format_kwargs):
from PseudoNetCDF import pncopen

keywords = {"kwargs": format_kwargs}
# only include mode if explicitly passed
Expand Down Expand Up @@ -138,3 +150,7 @@ def open_backend_dataset_pseudonetcdf(
open_dataset=open_backend_dataset_pseudonetcdf,
open_dataset_parameters=open_dataset_parameters,
)


if has_pseudonetcdf:
BACKEND_ENTRYPOINTS["pseudonetcdf"] = pseudonetcdf_backend
20 changes: 18 additions & 2 deletions xarray/backends/pydap_.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,22 @@
from ..core.pycompat import integer_types
from ..core.utils import Frozen, FrozenDict, close_on_error, is_dict_like, is_remote_uri
from ..core.variable import Variable
from .common import AbstractDataStore, BackendArray, BackendEntrypoint, robust_getitem
from .common import (
BACKEND_ENTRYPOINTS,
AbstractDataStore,
BackendArray,
BackendEntrypoint,
robust_getitem,
)
from .store import open_backend_dataset_store

try:
import pydap.client

has_pydap = True
except ModuleNotFoundError:
has_pydap = False


class PydapArrayWrapper(BackendArray):
def __init__(self, array):
Expand Down Expand Up @@ -74,7 +87,6 @@ def __init__(self, ds):

@classmethod
def open(cls, url, session=None):
import pydap.client

ds = pydap.client.open_url(url, session=session)
return cls(ds)
Expand Down Expand Up @@ -133,3 +145,7 @@ def open_backend_dataset_pydap(
pydap_backend = BackendEntrypoint(
open_dataset=open_backend_dataset_pydap, guess_can_open=guess_can_open_pydap
)


if has_pydap:
BACKEND_ENTRYPOINTS["pydap"] = pydap_backend
20 changes: 18 additions & 2 deletions xarray/backends/pynio_.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,24 @@
from ..core import indexing
from ..core.utils import Frozen, FrozenDict, close_on_error
from ..core.variable import Variable
from .common import AbstractDataStore, BackendArray, BackendEntrypoint
from .common import (
BACKEND_ENTRYPOINTS,
AbstractDataStore,
BackendArray,
BackendEntrypoint,
)
from .file_manager import CachingFileManager
from .locks import HDF5_LOCK, NETCDFC_LOCK, SerializableLock, combine_locks, ensure_lock
from .store import open_backend_dataset_store

try:
import Nio

has_pynio = True
except ModuleNotFoundError:
has_pynio = False


# PyNIO can invoke netCDF libraries internally
# Add a dedicated lock just in case NCL as well isn't thread-safe.
NCL_LOCK = SerializableLock()
Expand Down Expand Up @@ -45,7 +58,6 @@ class NioDataStore(AbstractDataStore):
"""Store for accessing datasets via PyNIO"""

def __init__(self, filename, mode="r", lock=None, **kwargs):
import Nio

if lock is None:
lock = PYNIO_LOCK
Expand Down Expand Up @@ -119,3 +131,7 @@ def open_backend_dataset_pynio(


pynio_backend = BackendEntrypoint(open_dataset=open_backend_dataset_pynio)


if has_pynio:
BACKEND_ENTRYPOINTS["pynio"] = pynio_backend
20 changes: 17 additions & 3 deletions xarray/backends/scipy_.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,24 @@
from ..core.indexing import NumpyIndexingAdapter
from ..core.utils import Frozen, FrozenDict, close_on_error, read_magic_number
from ..core.variable import Variable
from .common import BackendArray, BackendEntrypoint, WritableCFDataStore
from .common import (
BACKEND_ENTRYPOINTS,
BackendArray,
BackendEntrypoint,
WritableCFDataStore,
)
from .file_manager import CachingFileManager, DummyFileManager
from .locks import ensure_lock, get_write_lock
from .netcdf3 import encode_nc3_attr_value, encode_nc3_variable, is_valid_nc3_name
from .store import open_backend_dataset_store

try:
import scipy.io

has_scipy = True
except ModuleNotFoundError:
has_scipy = False


def _decode_string(s):
if isinstance(s, bytes):
Expand Down Expand Up @@ -61,8 +73,6 @@ def __setitem__(self, key, value):
def _open_scipy_netcdf(filename, mode, mmap, version):
import gzip

import scipy.io

# if the string ends with .gz, then gunzip and open as netcdf file
if isinstance(filename, str) and filename.endswith(".gz"):
try:
Expand Down Expand Up @@ -271,3 +281,7 @@ def open_backend_dataset_scipy(
scipy_backend = BackendEntrypoint(
open_dataset=open_backend_dataset_scipy, guess_can_open=guess_can_open_scipy
)


if has_scipy:
BACKEND_ENTRYPOINTS["scipy"] = scipy_backend
5 changes: 4 additions & 1 deletion xarray/backends/store.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from .. import conventions
from ..core.dataset import Dataset
from .common import AbstractDataStore, BackendEntrypoint
from .common import BACKEND_ENTRYPOINTS, AbstractDataStore, BackendEntrypoint


def guess_can_open_store(store_spec):
Expand Down Expand Up @@ -44,3 +44,6 @@ def open_backend_dataset_store(
store_backend = BackendEntrypoint(
open_dataset=open_backend_dataset_store, guess_can_open=guess_can_open_store
)


BACKEND_ENTRYPOINTS["store"] = store_backend
Loading

0 comments on commit 7dbbdca

Please sign in to comment.