Skip to content

Commit

Permalink
Merge pull request #307 from davidhassell/h5-read
Browse files Browse the repository at this point in the history
h5netcdf read
  • Loading branch information
davidhassell authored Jul 25, 2024
2 parents 4106b44 + 6f1c397 commit 9ceb39e
Show file tree
Hide file tree
Showing 52 changed files with 5,723 additions and 1,188 deletions.
18 changes: 17 additions & 1 deletion Changelog.rst
Original file line number Diff line number Diff line change
@@ -1,13 +1,29 @@
Version NEXTRELEASE
Version NEXTVERSION
-------------------

**2024-??-??**

* Upgrades to allow cfdm to work with Python 3.12
(https://github.com/NCAS-CMS/cfdm/issues/302)
* New function `cfdm.netcdf_flattener` that replaces the import of
`netcdf_flattener` (https://github.com/NCAS-CMS/cfdm/issues/286)
* New function `cfdm.netcdf_indexer` that applies netCDF masking and
unpacking to arbitrary arrays
(https://github.com/NCAS-CMS/cfdm/issues/285)
* Allow access to netCDF-4 files in S3 object stores
(https://github.com/NCAS-CMS/cfdm/issues/285)
* Allow a choice of netCDF engines
(https://github.com/NCAS-CMS/cfdm/issues/285)
* Fix bug that caused `cfdm.write` to fail when a parametric Z
dimension coordinate did not have a ``computed_standard_name``
attribute (https://github.com/NCAS-CMS/cfdm/issues/303)
* New class `cfdm.H5netcdfArray`
* New class `cfdm.NetCDF4Array`
* New dependency: ``h5netcdf>=1.3.0``
* New dependency: ``h5py>=3.10.0``
* New dependency: ``s3fs>=2024.6.0``
* New dependency: ``dask>=2024.6.0``
* Removed dependency: ``netcdf_flattener``

----

Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ inspecting it:

The ``cfdm`` package can:

* read field and domain constructs from netCDF and CDL datasets,
* read field and domain constructs from netCDF and CDL datasets with a
choice of netCDF backends,
* create new field and domain constructs in memory,
* write and append field and domain constructs to netCDF datasets on disk,
* read, write, and manipulate UGRID mesh topologies,
Expand Down
86 changes: 73 additions & 13 deletions cfdm/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,49 +48,105 @@
__cf_version__ = core.__cf_version__
__version__ = core.__version__

_requires = ("cftime", "netcdf_flattener", "scipy")
_requires = core._requires + (
"cftime",
"netCDF4",
"scipy",
"h5netcdf",
"s3fs",
)

_error0 = f"cfdm requires the modules {', '.join(_requires)}. "

# Check the version of cftime
try:
import cftime
except ImportError as error1:
raise ImportError(_error0 + str(error1))

# Check the version of cftime
_minimum_vn = "1.6.0"
if Version(cftime.__version__) < Version(_minimum_vn):
raise ValueError(
f"Bad cftime version: cfdm requires cftime>={_minimum_vn}. "
f"Got {cftime.__version__} at {cftime.__file__}"
)

# Check the version of netCDF4
try:
import netcdf_flattener
import netCDF4
except ImportError as error1:
raise ImportError(_error0 + str(error1))

# Check the version of netcdf_flattener
_minimum_vn = "1.2.0"
if Version(netcdf_flattener.__version__) < Version(_minimum_vn):
_minimum_vn = "1.5.4"
if Version(netCDF4.__version__) < Version(_minimum_vn):
raise ValueError(
f"Bad netcdf_flattener version: cfdm requires "
f"netcdf_flattener>={_minimum_vn}. Got {netcdf_flattener.__version__} "
f"at {netcdf_flattener.__file__}"
f"Bad netCDF4 version: cfdm requires netCDF4>={_minimum_vn}. "
f"Got {netCDF4.__version__} at {netCDF4.__file__}"
)

# Check the version of h5netcdf
try:
import scipy
import h5netcdf
except ImportError as error1:
raise ImportError(_error0 + str(error1))

_minimum_vn = "1.3.0"
if Version(h5netcdf.__version__) < Version(_minimum_vn):
raise ValueError(
f"Bad h5netcdf version: cfdm requires h5netcdf>={_minimum_vn}. "
f"Got {h5netcdf.__version__} at {h5netcdf.__file__}"
)

# Check the version of h5py
try:
import h5py
except ImportError as error1:
raise ImportError(_error0 + str(error1))

_minimum_vn = "3.10.0"
if Version(h5py.__version__) < Version(_minimum_vn):
raise ValueError(
f"Bad h5py version: cfdm requires h5py>={_minimum_vn}. "
f"Got {h5py.__version__} at {h5py.__file__}"
)

# Check the version of s3fs
try:
import s3fs
except ImportError as error1:
raise ImportError(_error0 + str(error1))

_minimum_vn = "2024.6.0"
if Version(s3fs.__version__) < Version(_minimum_vn):
raise ValueError(
f"Bad s3fs version: cfdm requires s3fs>={_minimum_vn}. "
f"Got {s3fs.__version__} at {s3fs.__file__}"
)

# Check the version of scipy
try:
import scipy
except ImportError as error1:
raise ImportError(_error0 + str(error1))

_minimum_vn = "1.10.0"
if Version(scipy.__version__) < Version(_minimum_vn):
raise ValueError(
f"Bad scipy version: cfdm requires "
f"scipy>={_minimum_vn}. Got {scipy.__version__} "
f"at {scipy.__file__}"
f"Bad scipy version: cfdm requires scipy>={_minimum_vn}. "
f"Got {scipy.__version__} at {scipy.__file__}"
)

# Check the version of dask
try:
import dask
except ImportError as error1:
raise ImportError(_error0 + str(error1))

_minimum_vn = "2024.6.0"
if Version(dask.__version__) < Version(_minimum_vn):
raise ValueError(
f"Bad scipy version: cfdm requires dask>={_minimum_vn}. "
f"Got {dask.__version__} at {dask.__file__}"
)

from .constants import masked
Expand Down Expand Up @@ -140,7 +196,10 @@
CompressedArray,
Data,
GatheredArray,
H5netcdfArray,
NetCDFArray,
NetCDF4Array,
netcdf_indexer,
NumpyArray,
PointTopologyArray,
RaggedArray,
Expand Down Expand Up @@ -196,6 +255,7 @@
from .cfdmimplementation import CFDMImplementation, implementation

from .read_write import read, write
from .read_write.netcdf.flatten import netcdf_flatten

from .examplefield import example_field, example_fields, example_domain

Expand Down
2 changes: 1 addition & 1 deletion cfdm/abstract/implementation.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def classes(self):
'Index',
'InteriorRing',
'List',
'NetCDFArray',
'NetCDF4Array',
'NodeCountProperties',
'PartNodeCountProperties',
'RaggedContiguousArray',
Expand Down
81 changes: 29 additions & 52 deletions cfdm/cfdmimplementation.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@
CellConnectivityArray,
Data,
GatheredArray,
NetCDFArray,
H5netcdfArray,
NetCDF4Array,
PointTopologyArray,
RaggedContiguousArray,
RaggedIndexedArray,
Expand Down Expand Up @@ -1358,10 +1359,10 @@ def get_data_maximum(self, parent):
:Returns:
Data instance
Scalar `Data` instance
"""
return parent.data.maximum()
return parent.data.maximum(squeeze=True)

def get_data_sum(self, parent):
"""Return the sum of the data.
Expand All @@ -1372,10 +1373,10 @@ def get_data_sum(self, parent):
:Returns:
Data instance
Scalar `Data` instance
"""
return parent.data.sum()
return parent.data.sum(squeeze=True)

def get_count(self, construct):
"""Return the count variable of compressed data.
Expand Down Expand Up @@ -2291,67 +2292,41 @@ def initialise_TiePointIndex(self):
cls = self.get_class("TiePointIndex")
return cls()

def initialise_NetCDFArray(
self,
filename=None,
address=None,
dtype=None,
shape=None,
mask=True,
units=False,
calendar=None,
missing_values=None,
):
"""Return a netCDF array instance.
def initialise_NetCDF4Array(self, **kwargs):
"""Return a `NetCDF4Array` instance.
:Parameters:
filename: `str`
address: `str`
dytpe: `numpy.dtype`
kwargs: optional
Initialisation parameters to pass to the new instance.
shape: sequence of `int`, optional
.. versionadded:: (cfdm) NEXTVERSION
mask: `bool`, optional
:Returns:
units: `str` or `None` or False, optional
The units of the netCDF variable. Set to `None` to
indicate that there are no units. If False (the
default) then the units are considered unset.
`NetCDF4Array`
.. versionadded:: (cfdm) 1.10.0.2
"""
cls = self.get_class("NetCDF4Array")
return cls(**kwargs)

calendar: `str` or `None`, optional
The calendar of the netCDF variable. By default, or if
set to `None`, then the CF default calendar is
assumed, if applicable.
def initialise_H5netcdfArray(self, **kwargs):
"""Return a `H5netcdfArray` instance.
.. versionadded:: (cfdm) 1.10.0.2
.. versionadded:: (cfdm) NEXTVERSION
missing_values: `dict`, optional
The missing value indicators defined by the netCDF
variable attributes.
:Parameters:
.. versionadded:: (cfdm) 1.10.0.3
kwargs: optional
Initialisation parameters to pass to the new instance.
:Returns:
`NetCDFArray`
`H5netcdfArray`
"""
cls = self.get_class("NetCDFArray")
return cls(
filename=filename,
address=address,
dtype=dtype,
shape=shape,
mask=mask,
units=units,
calendar=calendar,
missing_values=missing_values,
)
cls = self.get_class("H5netcdfArray")
return cls(**kwargs)

def initialise_BoundsFromNodesArray(self, **kwargs):
"""Return a node bounds array.
Expand Down Expand Up @@ -3707,7 +3682,8 @@ def squeeze(self, construct, axes=None):
Data=Data,
BoundsFromNodesArray=BoundsFromNodesArray,
GatheredArray=GatheredArray,
NetCDFArray=NetCDFArray,
H5netcdfArray=H5netcdfArray,
NetCDF4Array=NetCDF4Array,
PointTopologyArray=PointTopologyArray,
RaggedContiguousArray=RaggedContiguousArray,
RaggedIndexedArray=RaggedIndexedArray,
Expand Down Expand Up @@ -3750,7 +3726,8 @@ def implementation():
'Datum': <class 'cfdm.datum.Datum'>,
'Data': <class 'cfdm.data.data.Data'>,
'GatheredArray': <class 'cfdm.data.gatheredarray.GatheredArray'>,
'NetCDFArray': <class 'cfdm.data.netcdfarray.NetCDFArray'>,
'H5netcdfArray': <class 'cfdm.data.h5netcdfarray.H5netcdfArray'>,
'NetCDF4Array': <class 'cfdm.data.netcdf4array.NetCDF4Array'>,
'PointTopologyArray': <class 'cfdm.data.pointtopologyarray.PointTopologyArray'>,
'RaggedContiguousArray': <class 'cfdm.data.raggedcontiguousarray.RaggedContiguousArray'>,
'RaggedIndexedArray': <class 'cfdm.data.raggedindexedarray.RaggedIndexedArray'>,
Expand Down
36 changes: 14 additions & 22 deletions cfdm/core/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,30 +11,20 @@
"""

__date__ = "2024-03-01"
__date__ = "2024-??-??"
__cf_version__ = "1.11"
__version__ = "1.11.1.0"
__version__ = "1.11.2.0"

from packaging import __version__ as _packaging_ver
from packaging import __file__ as _packaging_file
from packaging.version import Version

import platform

_requires = ("numpy", "netCDF4", "packaging")
_requires = ("numpy", "packaging")

_error0 = f"cfdm.core requires the modules {', '.join(_requires)}. "

try:
import netCDF4
except ImportError as error1:
raise ImportError(_error0 + str(error1))

try:
import numpy as np
except ImportError as error1:
raise ImportError(_error0 + str(error1))

# Check the version of python
_minimum_vn = "3.8.0"
if Version(platform.python_version()) < Version(_minimum_vn):
Expand All @@ -44,22 +34,24 @@
)

# Check the version of packaging
try:
import packaging
except ImportError as error1:
raise ImportError(_error0 + str(error1))

_minimum_vn = "20.0"
if Version(_packaging_ver) < Version(_minimum_vn):
raise ValueError(
f"Bad packaging version: cfdm requires packaging>={_minimum_vn}. "
f"Bad packaging version: cfdm.core requires packaging>={_minimum_vn}. "
f"Got {_packaging_ver} at {_packaging_file}"
)

# Check the version of netCDF4
_minimum_vn = "1.5.4"
if Version(netCDF4.__version__) < Version(_minimum_vn):
raise ValueError(
f"Bad netCDF4 version: cfdm.core requires netCDF4>={_minimum_vn}. "
f"Got {netCDF4.__version__} at {netCDF4.__file__}"
)

# Check the version of numpy
try:
import numpy as np
except ImportError as error1:
raise ImportError(_error0 + str(error1))

_minimum_vn = "1.15"
if Version(np.__version__) < Version(_minimum_vn):
raise ValueError(
Expand Down
Loading

0 comments on commit 9ceb39e

Please sign in to comment.