diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88c1ea5ac0..efe02bac42 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: # Use specific format-enforcing pre-commit hooks from the core library # with the default configuration (see pre-commit.com for documentation) - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.3.0 hooks: - id: check-ast - id: debug-statements @@ -23,7 +23,7 @@ repos: # (see https://black.readthedocs.io/en/stable/ for documentation and see # the cf-python pyproject.toml file for our custom black configuration) - repo: https://github.com/ambv/black - rev: 21.5b0 + rev: 22.3.0 hooks: - id: black language_version: python3 @@ -48,7 +48,7 @@ repos: # (see https://flake8.pycqa.org/en/latest/ for documentation and see # the cf-python .flake8 file for our custom flake8 configuration) - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.1 + rev: 3.9.2 hooks: - id: flake8 @@ -57,7 +57,7 @@ repos: # compatible with 'black' with the lines set to ensure so in the repo's # pyproject.toml. Other than that and the below, no extra config is required. - repo: https://github.com/pycqa/isort - rev: 5.8.0 + rev: 5.10.1 hooks: - id: isort name: isort (python) diff --git a/cf/__init__.py b/cf/__init__.py index 99fbb6cd22..aadf7fd478 100644 --- a/cf/__init__.py +++ b/cf/__init__.py @@ -185,7 +185,7 @@ ) # Check the version of dask -_minimum_vn = "2022.03.0" +_minimum_vn = "2022.6.0" if LooseVersion(dask.__version__) < LooseVersion(_minimum_vn): raise RuntimeError( f"Bad dask version: cf requires dask>={_minimum_vn}. " diff --git a/cf/data/collapse.py b/cf/data/collapse.py index 0e71ff8777..f705e1eff5 100644 --- a/cf/data/collapse.py +++ b/cf/data/collapse.py @@ -1137,11 +1137,7 @@ def cf_mean_chunk(x, weights=None, dtype="f8", computing_meta=False, **kwargs): def cf_mean_combine( - pairs, - axis=None, - dtype="f8", - computing_meta=False, - **kwargs, + pairs, axis=None, dtype="f8", computing_meta=False, **kwargs ): """Combination calculations for the mean. @@ -1255,12 +1251,7 @@ def cf_max_chunk(x, dtype=None, computing_meta=False, **kwargs): } -def cf_max_combine( - pairs, - axis=None, - computing_meta=False, - **kwargs, -): +def cf_max_combine(pairs, axis=None, computing_meta=False, **kwargs): """Combination calculations for the maximum. This function is passed to `dask.array.reduction` as its *combine* @@ -1284,10 +1275,7 @@ def cf_max_combine( if computing_meta: return mx - return { - "max": mx, - "N": sum_sample_sizes(pairs, axis, **kwargs), - } + return {"max": mx, "N": sum_sample_sizes(pairs, axis, **kwargs)} def cf_max_agg( @@ -1414,12 +1402,7 @@ def cf_min_chunk(x, dtype=None, computing_meta=False, **kwargs): } -def cf_min_combine( - pairs, - axis=None, - computing_meta=False, - **kwargs, -): +def cf_min_combine(pairs, axis=None, computing_meta=False, **kwargs): """Combination calculations for the minimum. This function is passed to `dask.array.reduction` as its *combine* @@ -1443,10 +1426,7 @@ def cf_min_combine( if computing_meta: return mn - return { - "min": mn, - "N": sum_sample_sizes(pairs, axis, **kwargs), - } + return {"min": mn, "N": sum_sample_sizes(pairs, axis, **kwargs)} def cf_min_agg( @@ -1528,11 +1508,7 @@ def cf_range_chunk(x, dtype=None, computing_meta=False, **kwargs): def cf_range_combine( - pairs, - axis=None, - dtype=None, - computing_meta=False, - **kwargs, + pairs, axis=None, dtype=None, computing_meta=False, **kwargs ): """Combination calculations for the range. @@ -1559,11 +1535,7 @@ def cf_range_combine( mn = min_arrays(pairs, "min", axis, None, **kwargs) - return { - "max": mx, - "min": mn, - "N": sum_sample_sizes(pairs, axis, **kwargs), - } + return {"max": mx, "min": mn, "N": sum_sample_sizes(pairs, axis, **kwargs)} def cf_range_agg( @@ -1724,11 +1696,7 @@ def cf_sample_size_chunk(x, dtype="i8", computing_meta=False, **kwargs): def cf_sample_size_combine( - pairs, - axis=None, - dtype="i8", - computing_meta=False, - **kwargs, + pairs, axis=None, dtype="i8", computing_meta=False, **kwargs ): """Combination calculations for the sample size. @@ -1836,11 +1804,7 @@ def cf_sum_chunk(x, weights=None, dtype="f8", computing_meta=False, **kwargs): def cf_sum_combine( - pairs, - axis=None, - dtype="f8", - computing_meta=False, - **kwargs, + pairs, axis=None, dtype="f8", computing_meta=False, **kwargs ): """Combination calculations for the sum. @@ -1865,10 +1829,7 @@ def cf_sum_combine( if computing_meta: return x - return { - "sum": x, - "N": sum_sample_sizes(pairs, axis, **kwargs), - } + return {"sum": x, "N": sum_sample_sizes(pairs, axis, **kwargs)} def cf_sum_agg( @@ -2099,11 +2060,7 @@ def cf_var_chunk( def cf_var_combine( - pairs, - axis=None, - dtype="f8", - computing_meta=False, - **kwargs, + pairs, axis=None, dtype="f8", computing_meta=False, **kwargs ): """Combination calculations for the variance. diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 73dc7aed8e..0771355e48 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -14,8 +14,7 @@ from ..cfdatetime import dt2rt, rt2dt from ..functions import atol as cf_atol from ..functions import rtol as cf_rtol - -# from dask.utils import deepmap # Apply function inside nested lists +from ..units import Units def _da_ma_allclose(x, y, masked_equal=True, rtol=None, atol=None): @@ -94,26 +93,14 @@ def allclose(a_blocks, b_blocks, rtol=rtol, atol=atol): for a, b in zip(flatten(a_blocks), flatten(b_blocks)): result &= np.ma.allclose( - a, - b, - masked_equal=masked_equal, - rtol=rtol, - atol=atol, + a, b, masked_equal=masked_equal, rtol=rtol, atol=atol ) return result axes = tuple(range(x.ndim)) return da.blockwise( - allclose, - "", - x, - axes, - y, - axes, - dtype=bool, - rtol=rtol, - atol=atol, + allclose, "", x, axes, y, axes, dtype=bool, rtol=rtol, atol=atol ) @@ -594,3 +581,45 @@ def cf_dt2rt(a, units): """ return dt2rt(a, units_out=units, units_in=None) + + +def cf_units(a, from_units, to_units): + """Convert array values to have different equivalent units. + + .. versionadded:: TODODASK + + .. seealso:: `cf.Data.Units` + + :Parameters: + + a: `numpy.ndarray` + The array. + + from_units: `Units` + The existing units of the array. + + to_units: `Units` + The units that the array should be converted to. Must be + equivalent to *from_units*. + + :Returns: + + `numpy.ndarray` + An array containing values in the new units. In order to + represent the new units, the returned data type may be + different from that of the input array. For instance, if + *a* has an integer data type, *from_units* are kilometres, + and *to_units* are ``'miles'`` then the returned array + will have a float data type. + + **Examples** + + >>> import numpy as np + >>> a = np.array([1, 2]) + >>> print(cf.data.dask_utils.cf_units(a, cf.Units('km'), cf.Units('m'))) + [1000. 2000.] + + """ + return Units.conform( + a, from_units=from_units, to_units=to_units, inplace=False + ) diff --git a/cf/data/data.py b/cf/data/data.py index 6980b0c6a5..f2bdb9eaa7 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -27,12 +27,9 @@ ) from ..functions import ( _DEPRECATION_ERROR_KWARGS, - _numpy_isclose, _section, - abspath, atol, default_netCDF_fillvals, - fm_threshold, free_memory, log_level, parse_indices, @@ -40,7 +37,6 @@ ) from ..mixin_container import Container from ..units import Units -from . import FileArray from .collapse import Collapse from .creation import compressed_to_dask, generate_axis_identifiers, to_dask from .dask_utils import ( @@ -51,6 +47,7 @@ cf_percentile, cf_rt2dt, cf_soften_mask, + cf_units, cf_where, ) from .mixin import DataClassDeprecationsMixin @@ -135,7 +132,7 @@ def wrapper(*args, **kwargs): _DEFAULT_HARDMASK = True -class Data(Container, cfdm.Data, DataClassDeprecationsMixin): +class Data(DataClassDeprecationsMixin, Container, cfdm.Data): """An N-dimensional data array with units and masked values. * Contains an N-dimensional, indexable and broadcastable array with @@ -406,19 +403,12 @@ def __init__( except (AttributeError, TypeError): pass else: - self._set_dask( - array, - copy=copy, - delete_source=False, - reset_mask_hardness=False, - ) + self._set_dask(array, copy=copy, delete_source=False) else: self._del_dask(None) - # Note the mask hardness. It is safe to assume that if a - # dask array has been set, then it's mask hardness will be - # already baked into each chunk. - self._hardmask = getattr(source, "hardmask", _DEFAULT_HARDMASK) + # Set the mask hardness + self.hardmask = getattr(source, "hardmask", _DEFAULT_HARDMASK) return @@ -428,13 +418,8 @@ def __init__( units = Units(units, calendar=calendar) self._Units = units - # Note the mask hardness. This only records what we want the - # mask hardness to be, and is required in case this - # initialization does not set an array (i.e. array is None or - # _use_array is False). If a dask array is actually set later - # on, then the mask hardness will be set properly, i.e. it - # will be baked into each chunk. - self._hardmask = hardmask + # Set the mask hardness + self.hardmask = hardmask if array is None: return @@ -544,10 +529,7 @@ def __init__( self._Units = units # Store the dask array - self._set_dask(array, delete_source=False, reset_mask_hardness=False) - - # Set the mask hardness on each chunk. - self.hardmask = hardmask + self._set_dask(array, delete_source=False) # Override the data type if dtype is not None: @@ -990,7 +972,7 @@ def __getitem__(self, indices): # ------------------------------------------------------------ # Set the subspaced dask array # ------------------------------------------------------------ - new._set_dask(dx, reset_mask_hardness=False) + new._set_dask(dx) # ------------------------------------------------------------ # Get the axis identifiers for the subspace @@ -1128,7 +1110,10 @@ def __setitem__(self, indices, value): value = conform_units(value, self.Units) # Do the assignment - dx = self.to_dask_array() + + # Missing values could be affected, so make sure that the mask + # hardness has been applied. + dx = self.to_dask_array(apply_mask_hardness=True) dx[indices] = value # Unroll any axes that were rolled to enable a cyclic @@ -1137,11 +1122,6 @@ def __setitem__(self, indices, value): shifts = [-shift for shift in shifts] self.roll(shift=shifts, axis=roll_axes, inplace=True) - # Reset the mask hardness, otherwise it could be incorrect in - # the case that a chunk that was not a masked array is - # assigned missing values. - self._reset_mask_hardness() - # Remove a source array, on the grounds that we can't # guarantee its consistency with the updated dask array. self._del_Array(None) @@ -1264,14 +1244,12 @@ def __keepdims_indexing__(self): def __keepdims_indexing__(self, value): self._custom["__keepdims_indexing__"] = bool(value) - def _set_dask( - self, array, copy=False, delete_source=True, reset_mask_hardness=True - ): + def _set_dask(self, array, copy=False, delete_source=True): """Set the dask array. .. versionadded:: TODODASK - .. seealso:: `to_dask_array`, `_del_dask`, `_reset_mask_hardness` + .. seealso:: `to_dask_array`, `_del_dask` :Parameters: @@ -1287,12 +1265,6 @@ def _set_dask( exists, after setting the new dask array. By default a source array is deleted. - reset_mask_hardness: `bool`, optional - If False then do not reset the mask hardness after - setting the new dask array. By default the mask - hardness is re-applied, even if the mask hardness has - not changed. - :Returns: `None` @@ -1323,9 +1295,6 @@ def _set_dask( # guarantee its consistency with the new dask array. self._del_Array(None) - if reset_mask_hardness: - self._reset_mask_hardness() - def _del_dask(self, default=ValueError(), delete_source=True): """Remove the dask array. @@ -1381,20 +1350,6 @@ def _del_dask(self, default=ValueError(), delete_source=True): return out - def _reset_mask_hardness(self): - """Re-apply the mask hardness to the dask array. - - .. versionadded:: TODODASK - - .. seealso:: `hardmask`, `harden_mask`, `soften_mask` - - :Returns: - - `None` - - """ - self.hardmask = self.hardmask - @daskified(_DASKIFIED_VERBOSE) @_inplace_enabled(default=False) def diff(self, axis=-1, n=1, inplace=False): @@ -1479,7 +1434,7 @@ def diff(self, axis=-1, n=1, inplace=False): dx = self.to_dask_array() dx = da.diff(dx, axis=axis, n=n) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d @@ -1766,7 +1721,7 @@ def digitize( # Digitise the array dx = d.to_dask_array() dx = da.digitize(dx, bins, right=upper) - d._set_dask(dx, reset_mask_hardness=True) + d._set_dask(dx) d.override_units(_units_None, inplace=True) if return_bins: @@ -1782,13 +1737,7 @@ def digitize( @daskified(_DASKIFIED_VERBOSE) @_deprecated_kwarg_check("_preserve_partitions") - def median( - self, - axes=None, - squeeze=False, - mtol=1, - inplace=False, - ): + def median(self, axes=None, squeeze=False, mtol=1, inplace=False): """Calculate median values. Calculates the median value or the median values along axes. @@ -1830,11 +1779,7 @@ def median( """ return self.percentile( - 50, - axes=axes, - squeeze=squeeze, - mtol=mtol, - inplace=inplace, + 50, axes=axes, squeeze=squeeze, mtol=mtol, inplace=inplace ) @_inplace_enabled(default=False) @@ -1906,11 +1851,7 @@ def mean_of_upper_decile( d = _inplace_enabled_define_and_cleanup(self) p90 = d.percentile( - 90, - axes=axes, - squeeze=False, - mtol=mtol, - inplace=False, + 90, axes=axes, squeeze=False, mtol=mtol, inplace=False ) with np.testing.suppress_warnings() as sup: @@ -2220,7 +2161,7 @@ def percentile( graph = HighLevelGraph.from_collections(name, dsk, dependencies=[dx]) dx = Array(graph, name, chunks=out_chunks, dtype=float) - d._set_dask(dx, reset_mask_hardness=True) + d._set_dask(dx) return d @@ -2266,7 +2207,7 @@ def persist(self, inplace=False): dx = self.to_dask_array() dx = dx.persist() - d._set_dask(dx, delete_source=False, reset_mask_hardness=False) + d._set_dask(dx, delete_source=False) return d @@ -2418,7 +2359,7 @@ def ceil(self, inplace=False, i=False): """ d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() - d._set_dask(da.ceil(dx), reset_mask_hardness=False) + d._set_dask(da.ceil(dx)) return d @daskified(_DASKIFIED_VERBOSE) @@ -2661,7 +2602,7 @@ def convolution_filter( meta=np.array((), dtype=float), ) - d._set_dask(dx, reset_mask_hardness=True) + d._set_dask(dx) return d @@ -2756,11 +2697,7 @@ def cumsum( dx = d.to_dask_array() dx = dx.cumsum(axis=axis, method=method) - - # Note: The dask cumsum method resets the mask hardness to the - # numpy default, so we need to reset the mask hardness - # during _set_dask. - d._set_dask(dx, reset_mask_hardness=True) + d._set_dask(dx) return d @@ -2845,7 +2782,7 @@ def rechunk( dx = d.to_dask_array() dx = dx.rechunk(chunks, threshold, block_size_limit, balance) - d._set_dask(dx, delete_source=False, reset_mask_hardness=False) + d._set_dask(dx, delete_source=False) return d @@ -2899,7 +2836,7 @@ def _asdatetime(self, inplace=False): if not d._isdatetime(): dx = d.to_dask_array() dx = dx.map_blocks(cf_rt2dt, units=units, dtype=object) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d @@ -2956,7 +2893,7 @@ def _asreftime(self, inplace=False): if d._isdatetime(): dx = d.to_dask_array() dx = dx.map_blocks(cf_dt2rt, units=units, dtype=float) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d @@ -3233,13 +3170,13 @@ def _combined_units(self, data1, method, inplace): # raising this to a power is a nonlinear # operation p = data0.datum(0) - if units0 != (units0 ** p) ** (1.0 / p): + if units0 != (units0**p) ** (1.0 / p): raise ValueError( "Can't raise shifted units {!r} to the " "power {}".format(units0, p) ) - return data0, data1, units1 ** p + return data0, data1, units1**p elif units0.isdimensionless: # units0 is dimensionless if not units0.equals(_units_1): @@ -3251,17 +3188,17 @@ def _combined_units(self, data1, method, inplace): # raising this to a power is a nonlinear # operation p = data0.datum(0) - if units0 != (units0 ** p) ** (1.0 / p): + if units0 != (units0**p) ** (1.0 / p): raise ValueError( "Can't raise shifted units {!r} to the " "power {}".format(units0, p) ) - return data0, data1, units1 ** p + return data0, data1, units1**p # --- End: if # This will deliberately raise an exception - units1 ** units0 + units1**units0 else: # ----------------------------------------------------- # Operator is __pow__ @@ -3309,13 +3246,13 @@ def _combined_units(self, data1, method, inplace): # raising this to a power is a nonlinear # operation p = data1.datum(0) - if units0 != (units0 ** p) ** (1.0 / p): + if units0 != (units0**p) ** (1.0 / p): raise ValueError( "Can't raise shifted units {!r} to the " "power {}".format(units0, p) ) - return data0, data1, units0 ** p + return data0, data1, units0**p elif units1.isdimensionless: # units1 is dimensionless if not units1.equals(_units_1): @@ -3326,17 +3263,17 @@ def _combined_units(self, data1, method, inplace): # raising this to a power is a nonlinear # operation p = data1.datum(0) - if units0 != (units0 ** p) ** (1.0 / p): + if units0 != (units0**p) ** (1.0 / p): raise ValueError( "Can't raise shifted units {!r} to the " "power {}".format(units0, p) ) - return data0, data1, units0 ** p + return data0, data1, units0**p # --- End: if # This will deliberately raise an exception - units0 ** units1 + units0**units1 # --- End: if # --- End: if @@ -3347,6 +3284,7 @@ def _combined_units(self, data1, method, inplace): ) ) + @daskified(_DASKIFIED_VERBOSE) def _binary_operation(self, other, method): """Implement binary arithmetic and comparison operations with the numpy broadcasting rules. @@ -3390,13 +3328,12 @@ def _binary_operation(self, other, method): """ inplace = method[2] == "i" - method_type = method[-5:-2] # ------------------------------------------------------------ - # Ensure that other is an independent Data object + # Ensure other is an independent Data object, for example + # so that combination with cf.Query objects works. # ------------------------------------------------------------ if getattr(other, "_NotImplemented_RHS_Data_op", False): - # Make sure that return NotImplemented elif not isinstance(other, self.__class__): @@ -3406,9 +3343,7 @@ def _binary_operation(self, other, method): and self.Units.isreftime ): other = cf_dt( - other, - # .timetuple()[0:6], microsecond=other.microsecond, - calendar=getattr(self.Units, "calendar", "standard"), + other, calendar=getattr(self.Units, "calendar", "standard") ) elif other is None: # Can't sensibly initialize a Data object from a bare @@ -3417,304 +3352,52 @@ def _binary_operation(self, other, method): other = type(self).asdata(other) - data0 = self.copy() - - data0, other, new_Units = data0._combined_units(other, method, True) - - # ------------------------------------------------------------ - # Bring other into memory, if appropriate. # ------------------------------------------------------------ - other.to_memory() - + # Prepare data0 (i.e. self copied) and data1 (i.e. other) # ------------------------------------------------------------ - # Find which dimensions need to be broadcast in one or other - # of the arrays. - # - # Method: - # - # For each common dimension, the 'broadcast_indices' list - # will have a value of None if there is no broadcasting - # required (i.e. the two arrays have the same size along - # that dimension) or a value of slice(None) if broadcasting - # is required (i.e. the two arrays have the different sizes - # along that dimension and one of the sizes is 1). - # - # Example: - # - # If c.shape is (7,1,6,1,5) and d.shape is (6,4,1) then - # broadcast_indices will be - # [None,slice(None),slice(None)]. - # - # The indices to d which correspond to a partition of c, - # are the relevant subset of partition.indices updated - # with the non None elements of the broadcast_indices - # list. - # - # In this example, if a partition of c were to have a - # partition.indices value of (slice(0,3), slice(0,1), - # slice(2,4), slice(0,1), slice(0,5)), then the relevant - # subset of these is partition.indices[2:] and the - # corresponding indices to d are (slice(2,4), slice(None), - # slice(None)) - # - # ------------------------------------------------------------ - data0_shape = data0._shape - data1_shape = other._shape - - if data0_shape == data1_shape: - # self and other have the same shapes - broadcasting = False - - align_offset = 0 - - new_shape = data0_shape - new_ndim = data0._ndim - new_axes = data0._axes - new_size = data0._size - - else: - # self and other have different shapes - broadcasting = True - - data0_ndim = data0._ndim - data1_ndim = other._ndim - - align_offset = data0_ndim - data1_ndim - if align_offset >= 0: - # self has at least as many axes as other - shape0 = data0_shape[align_offset:] - shape1 = data1_shape - - new_shape = data0_shape[:align_offset] - new_ndim = data0_ndim - new_axes = data0._axes - else: - # other has more axes than self - align_offset = -align_offset - shape0 = data0_shape - shape1 = data1_shape[align_offset:] - - new_shape = data1_shape[:align_offset] - new_ndim = data1_ndim - if not data0_ndim: - new_axes = other._axes - else: - new_axes = [] - existing_axes = self._all_axis_names() - for n in new_shape: - axis = new_axis_identifier(existing_axes) - existing_axes.append(axis) - new_axes.append(axis) - # --- End: for - new_axes += data0._axes - # --- End: for - - align_offset = 0 - # --- End: if - - broadcast_indices = [] - for a, b in zip(shape0, shape1): - if a == b: - new_shape += (a,) - broadcast_indices.append(None) - continue - - # Still here? - if a > 1 and b == 1: - new_shape += (a,) - elif b > 1 and a == 1: - new_shape += (b,) - else: - raise ValueError( - "Can't broadcast shape {} against shape {}".format( - data1_shape, data0_shape - ) - ) - - broadcast_indices.append(slice(None)) - - new_size = reduce(mul, new_shape, 1) - - dummy_location = [None] * new_ndim - # ---End: if - - new_flip = [] + data0 = self.copy() - # ------------------------------------------------------------ - # Create a Data object which just contains the metadata for - # the result. If we're doing a binary arithmetic operation - # then result will get filled with data and returned. If we're - # an augmented arithmetic assignment then we'll update self - # with this new metadata. - # ------------------------------------------------------------ + # Parse units + data0, other, new_Units = data0._combined_units(other, method, True) - result = data0.copy() - result._shape = new_shape - result._ndim = new_ndim - result._size = new_size - result._axes = new_axes + # Cast as dask arrays + dx0 = data0.to_dask_array() + dx1 = other.to_dask_array() - # ------------------------------------------------------------ - # Set the data-type of the result - # ------------------------------------------------------------ - if method_type in ("_eq", "_ne", "_lt", "_le", "_gt", "_ge"): - new_dtype = np.dtype(bool) + # Set if applicable the tolerance levels for the result + if method in ("__eq__", "__ne__"): rtol = self._rtol atol = self._atol - else: - if "true" in method: - new_dtype = np.dtype(float) - elif not inplace: - new_dtype = np.result_type(data0.dtype, other.dtype) - else: - new_dtype = data0.dtype - # --- End: if # ------------------------------------------------------------ - # Set flags to control whether or not the data of result and - # self should be kept in memory + # Perform the binary operation with data0 (self) and data1 + # (other) # ------------------------------------------------------------ - config = data0.partition_configuration(readonly=not inplace) - - original_numpy_seterr = np.seterr(**_seterr) - - # Think about dtype, here. - - for partition_r, partition_s in zip( - result.partitions.matrix.flat, data0.partitions.matrix.flat - ): - - partition_s.open(config) - - indices = partition_s.indices - - array0 = partition_s.array - - if broadcasting: - indices = tuple( - [ - (index if not broadcast_index else broadcast_index) - for index, broadcast_index in zip( - indices[align_offset:], broadcast_indices - ) - ] - ) - indices = (Ellipsis,) + indices - - array1 = other[indices].array - - # UNRESOLVED ISSUE: array1 could be much larger than the - # chunk size. - - if not inplace: - partition = partition_r - partition.update_inplace_from(partition_s) + if method == "__eq__": + if dx0.dtype.kind in "US" or dx1.dtype.kind in "US": + result = getattr(dx0, method)(dx1) else: - partition = partition_s - - # -------------------------------------------------------- - # Do the binary operation on this partition's data - # -------------------------------------------------------- - try: - if method == "__eq__": # and data0.Units.isreftime: - array0 = _numpy_isclose( - array0, array1, rtol=rtol, atol=atol - ) - elif method == "__ne__": - array0 = ~_numpy_isclose( - array0, array1, rtol=rtol, atol=atol - ) - else: - array0 = getattr(array0, method)(array1) - - except FloatingPointError as error: - # Floating point point errors have been trapped - if _mask_fpe[0]: - # Redo the calculation ignoring the errors and - # then set invalid numbers to missing data - np.seterr(**_seterr_raise_to_ignore) - array0 = getattr(array0, method)(array1) - array0 = np.ma.masked_invalid(array0, copy=False) - np.seterr(**_seterr) - else: - # Raise the floating point error exception - raise FloatingPointError(error) - except TypeError as error: - if inplace: - raise TypeError( - "Incompatible result data-type ({0!r}) for " - "in-place {1!r} arithmetic".format( - np.result_type(array0.dtype, array1.dtype).name, - array0.dtype.name, - ) - ) - else: - raise TypeError(error) - # --- End: try - - if array0 is NotImplemented: - array0 = np.zeros(partition.shape, dtype=bool) - elif not array0.ndim and not isinstance(array0, np.ndarray): - array0 = np.asanyarray(array0) - - if not inplace: - p_datatype = array0.dtype - if new_dtype != p_datatype: - new_dtype = np.result_type(p_datatype, new_dtype) - - partition.subarray = array0 - partition.Units = new_Units - partition.axes = new_axes - partition.flip = new_flip - partition.part = [] - - if broadcasting: - partition.location = dummy_location - partition.shape = list(array0.shape) - - partition._original = None - partition._write_to_disk = False - partition.close(units=new_Units) - - if not inplace: - partition_s.close() - # --- End: for - - # Reset numpy.seterr - np.seterr(**original_numpy_seterr) - - source = result.source(None) - if source is not None and source.get_compression_type(): - result._del_Array(None) - - if not inplace: - result._Units = new_Units - result.dtype = new_dtype - result._flip(new_flip) - - if broadcasting: - result.partitions.set_location_map(result._axes) - - if method_type in ("_eq", "_ne", "_lt", "_le", "_gt", "_ge"): - result.override_units(Units(), inplace=True) - - return result + result = da.isclose(dx0, dx1, rtol=rtol, atol=atol) + elif method == "__ne__": + if dx0.dtype.kind in "US" or dx1.dtype.kind in "US": + result = getattr(dx0, method)(dx1) + else: + result = ~da.isclose(dx0, dx1, rtol=rtol, atol=atol) + elif inplace: + # Find non-in-place equivalent operator (remove 'i') + equiv_method = method[:2] + method[3:] + result = getattr(dx0, equiv_method)(dx1) else: - # Update the metadata for the new master array in place - data0._shape = new_shape - data0._ndim = new_ndim - data0._size = new_size - data0._axes = new_axes - data0._flip(new_flip) - data0._Units = new_Units - data0.dtype = new_dtype - - if broadcasting: - data0.partitions.set_location_map(new_axes) - - self.__dict__ = data0.__dict__ + result = getattr(dx0, method)(dx1) + if inplace: # in-place so concerns original self + self._set_dask(result) + self.override_units(new_Units, inplace=True) return self + else: # not, so concerns a new Data object copied from self, data0 + data0._set_dask(result) + data0.override_units(new_Units, inplace=True) + return data0 def _parse_indices(self, *args, **kwargs): """'cf.Data._parse_indices' is not available. @@ -4055,6 +3738,7 @@ def _move_flip_to_partitions(self): self._flip([]) + @daskified(_DASKIFIED_VERBOSE) def _unary_operation(self, operation): """Implement unary arithmetic operations. @@ -4095,10 +3779,11 @@ def _unary_operation(self, operation): dx = self.to_dask_array() dx = getattr(operator, operation)(dx) - out._set_dask(dx, reset_mask_hardness=False) + out._set_dask(dx) return out + @daskified(_DASKIFIED_VERBOSE) def __add__(self, other): """The binary arithmetic operation ``+`` @@ -4107,6 +3792,7 @@ def __add__(self, other): """ return self._binary_operation(other, "__add__") + @daskified(_DASKIFIED_VERBOSE) def __iadd__(self, other): """The augmented arithmetic assignment ``+=`` @@ -4115,6 +3801,7 @@ def __iadd__(self, other): """ return self._binary_operation(other, "__iadd__") + @daskified(_DASKIFIED_VERBOSE) def __radd__(self, other): """The binary arithmetic operation ``+`` with reflected operands. @@ -4124,6 +3811,7 @@ def __radd__(self, other): """ return self._binary_operation(other, "__radd__") + @daskified(_DASKIFIED_VERBOSE) def __sub__(self, other): """The binary arithmetic operation ``-`` @@ -4132,6 +3820,7 @@ def __sub__(self, other): """ return self._binary_operation(other, "__sub__") + @daskified(_DASKIFIED_VERBOSE) def __isub__(self, other): """The augmented arithmetic assignment ``-=`` @@ -4140,6 +3829,7 @@ def __isub__(self, other): """ return self._binary_operation(other, "__isub__") + @daskified(_DASKIFIED_VERBOSE) def __rsub__(self, other): """The binary arithmetic operation ``-`` with reflected operands. @@ -4149,6 +3839,7 @@ def __rsub__(self, other): """ return self._binary_operation(other, "__rsub__") + @daskified(_DASKIFIED_VERBOSE) def __mul__(self, other): """The binary arithmetic operation ``*`` @@ -4157,6 +3848,7 @@ def __mul__(self, other): """ return self._binary_operation(other, "__mul__") + @daskified(_DASKIFIED_VERBOSE) def __imul__(self, other): """The augmented arithmetic assignment ``*=`` @@ -4165,6 +3857,7 @@ def __imul__(self, other): """ return self._binary_operation(other, "__imul__") + @daskified(_DASKIFIED_VERBOSE) def __rmul__(self, other): """The binary arithmetic operation ``*`` with reflected operands. @@ -4174,6 +3867,7 @@ def __rmul__(self, other): """ return self._binary_operation(other, "__rmul__") + @daskified(_DASKIFIED_VERBOSE) def __div__(self, other): """The binary arithmetic operation ``/`` @@ -4182,6 +3876,7 @@ def __div__(self, other): """ return self._binary_operation(other, "__div__") + @daskified(_DASKIFIED_VERBOSE) def __idiv__(self, other): """The augmented arithmetic assignment ``/=`` @@ -4190,6 +3885,7 @@ def __idiv__(self, other): """ return self._binary_operation(other, "__idiv__") + @daskified(_DASKIFIED_VERBOSE) def __rdiv__(self, other): """The binary arithmetic operation ``/`` with reflected operands. @@ -4199,6 +3895,7 @@ def __rdiv__(self, other): """ return self._binary_operation(other, "__rdiv__") + @daskified(_DASKIFIED_VERBOSE) def __floordiv__(self, other): """The binary arithmetic operation ``//`` @@ -4207,6 +3904,7 @@ def __floordiv__(self, other): """ return self._binary_operation(other, "__floordiv__") + @daskified(_DASKIFIED_VERBOSE) def __ifloordiv__(self, other): """The augmented arithmetic assignment ``//=`` @@ -4215,6 +3913,7 @@ def __ifloordiv__(self, other): """ return self._binary_operation(other, "__ifloordiv__") + @daskified(_DASKIFIED_VERBOSE) def __rfloordiv__(self, other): """The binary arithmetic operation ``//`` with reflected operands. @@ -4224,6 +3923,7 @@ def __rfloordiv__(self, other): """ return self._binary_operation(other, "__rfloordiv__") + @daskified(_DASKIFIED_VERBOSE) def __truediv__(self, other): """The binary arithmetic operation ``/`` (true division) @@ -4232,6 +3932,7 @@ def __truediv__(self, other): """ return self._binary_operation(other, "__truediv__") + @daskified(_DASKIFIED_VERBOSE) def __itruediv__(self, other): """The augmented arithmetic assignment ``/=`` (true division) @@ -4240,6 +3941,7 @@ def __itruediv__(self, other): """ return self._binary_operation(other, "__itruediv__") + @daskified(_DASKIFIED_VERBOSE) def __rtruediv__(self, other): """The binary arithmetic operation ``/`` (true division) with reflected operands. @@ -4249,6 +3951,7 @@ def __rtruediv__(self, other): """ return self._binary_operation(other, "__rtruediv__") + @daskified(_DASKIFIED_VERBOSE) def __pow__(self, other, modulo=None): """The binary arithmetic operations ``**`` and ``pow`` @@ -4264,6 +3967,7 @@ def __pow__(self, other, modulo=None): return self._binary_operation(other, "__pow__") + @daskified(_DASKIFIED_VERBOSE) def __ipow__(self, other, modulo=None): """The augmented arithmetic assignment ``**=`` @@ -4279,6 +3983,7 @@ def __ipow__(self, other, modulo=None): return self._binary_operation(other, "__ipow__") + @daskified(_DASKIFIED_VERBOSE) def __rpow__(self, other, modulo=None): """The binary arithmetic operations ``**`` and ``pow`` with reflected operands. @@ -4295,6 +4000,7 @@ def __rpow__(self, other, modulo=None): return self._binary_operation(other, "__rpow__") + @daskified(_DASKIFIED_VERBOSE) def __mod__(self, other): """The binary arithmetic operation ``%`` @@ -4303,6 +4009,7 @@ def __mod__(self, other): """ return self._binary_operation(other, "__mod__") + @daskified(_DASKIFIED_VERBOSE) def __imod__(self, other): """The binary arithmetic operation ``%=`` @@ -4311,6 +4018,7 @@ def __imod__(self, other): """ return self._binary_operation(other, "__imod__") + @daskified(_DASKIFIED_VERBOSE) def __rmod__(self, other): """The binary arithmetic operation ``%`` with reflected operands. @@ -4320,6 +4028,7 @@ def __rmod__(self, other): """ return self._binary_operation(other, "__rmod__") + @daskified(_DASKIFIED_VERBOSE) def __eq__(self, other): """The rich comparison operator ``==`` @@ -4328,6 +4037,7 @@ def __eq__(self, other): """ return self._binary_operation(other, "__eq__") + @daskified(_DASKIFIED_VERBOSE) def __ne__(self, other): """The rich comparison operator ``!=`` @@ -4336,6 +4046,7 @@ def __ne__(self, other): """ return self._binary_operation(other, "__ne__") + @daskified(_DASKIFIED_VERBOSE) def __ge__(self, other): """The rich comparison operator ``>=`` @@ -4344,6 +4055,7 @@ def __ge__(self, other): """ return self._binary_operation(other, "__ge__") + @daskified(_DASKIFIED_VERBOSE) def __gt__(self, other): """The rich comparison operator ``>`` @@ -4352,6 +4064,7 @@ def __gt__(self, other): """ return self._binary_operation(other, "__gt__") + @daskified(_DASKIFIED_VERBOSE) def __le__(self, other): """The rich comparison operator ``<=`` @@ -4360,6 +4073,7 @@ def __le__(self, other): """ return self._binary_operation(other, "__le__") + @daskified(_DASKIFIED_VERBOSE) def __lt__(self, other): """The rich comparison operator ``<`` @@ -4368,6 +4082,7 @@ def __lt__(self, other): """ return self._binary_operation(other, "__lt__") + @daskified(_DASKIFIED_VERBOSE) def __and__(self, other): """The binary bitwise operation ``&`` @@ -4376,6 +4091,7 @@ def __and__(self, other): """ return self._binary_operation(other, "__and__") + @daskified(_DASKIFIED_VERBOSE) def __iand__(self, other): """The augmented bitwise assignment ``&=`` @@ -4384,6 +4100,7 @@ def __iand__(self, other): """ return self._binary_operation(other, "__iand__") + @daskified(_DASKIFIED_VERBOSE) def __rand__(self, other): """The binary bitwise operation ``&`` with reflected operands. @@ -4392,6 +4109,7 @@ def __rand__(self, other): """ return self._binary_operation(other, "__rand__") + @daskified(_DASKIFIED_VERBOSE) def __or__(self, other): """The binary bitwise operation ``|`` @@ -4400,6 +4118,7 @@ def __or__(self, other): """ return self._binary_operation(other, "__or__") + @daskified(_DASKIFIED_VERBOSE) def __ior__(self, other): """The augmented bitwise assignment ``|=`` @@ -4408,6 +4127,7 @@ def __ior__(self, other): """ return self._binary_operation(other, "__ior__") + @daskified(_DASKIFIED_VERBOSE) def __ror__(self, other): """The binary bitwise operation ``|`` with reflected operands. @@ -4416,6 +4136,7 @@ def __ror__(self, other): """ return self._binary_operation(other, "__ror__") + @daskified(_DASKIFIED_VERBOSE) def __xor__(self, other): """The binary bitwise operation ``^`` @@ -4424,6 +4145,7 @@ def __xor__(self, other): """ return self._binary_operation(other, "__xor__") + @daskified(_DASKIFIED_VERBOSE) def __ixor__(self, other): """The augmented bitwise assignment ``^=`` @@ -4432,6 +4154,7 @@ def __ixor__(self, other): """ return self._binary_operation(other, "__ixor__") + @daskified(_DASKIFIED_VERBOSE) def __rxor__(self, other): """The binary bitwise operation ``^`` with reflected operands. @@ -4440,6 +4163,7 @@ def __rxor__(self, other): """ return self._binary_operation(other, "__rxor__") + @daskified(_DASKIFIED_VERBOSE) def __lshift__(self, y): """The binary bitwise operation ``<<`` @@ -4448,6 +4172,7 @@ def __lshift__(self, y): """ return self._binary_operation(y, "__lshift__") + @daskified(_DASKIFIED_VERBOSE) def __ilshift__(self, y): """The augmented bitwise assignment ``<<=`` @@ -4456,6 +4181,7 @@ def __ilshift__(self, y): """ return self._binary_operation(y, "__ilshift__") + @daskified(_DASKIFIED_VERBOSE) def __rlshift__(self, y): """The binary bitwise operation ``<<`` with reflected operands. @@ -4464,6 +4190,7 @@ def __rlshift__(self, y): """ return self._binary_operation(y, "__rlshift__") + @daskified(_DASKIFIED_VERBOSE) def __rshift__(self, y): """The binary bitwise operation ``>>`` @@ -4472,6 +4199,7 @@ def __rshift__(self, y): """ return self._binary_operation(y, "__rshift__") + @daskified(_DASKIFIED_VERBOSE) def __irshift__(self, y): """The augmented bitwise assignment ``>>=`` @@ -4480,6 +4208,7 @@ def __irshift__(self, y): """ return self._binary_operation(y, "__irshift__") + @daskified(_DASKIFIED_VERBOSE) def __rrshift__(self, y): """The binary bitwise operation ``>>`` with reflected operands. @@ -4488,6 +4217,7 @@ def __rrshift__(self, y): """ return self._binary_operation(y, "__rrshift__") + @daskified(_DASKIFIED_VERBOSE) def __abs__(self): """The unary arithmetic operation ``abs`` @@ -4496,6 +4226,7 @@ def __abs__(self): """ return self._unary_operation("__abs__") + @daskified(_DASKIFIED_VERBOSE) def __neg__(self): """The unary arithmetic operation ``-`` @@ -4504,6 +4235,7 @@ def __neg__(self): """ return self._unary_operation("__neg__") + @daskified(_DASKIFIED_VERBOSE) def __invert__(self): """The unary bitwise operation ``~`` @@ -4512,6 +4244,7 @@ def __invert__(self): """ return self._unary_operation("__invert__") + @daskified(_DASKIFIED_VERBOSE) def __pos__(self): """The unary arithmetic operation ``+`` @@ -4580,35 +4313,6 @@ def _cyclic(self, value): def _cyclic(self): self._custom["_cyclic"] = _empty_set - @property - @daskified(_DASKIFIED_VERBOSE) - def _hardmask(self): - """Storage for the mask hardness. - - Contains a `bool`, where `True` denotes a hard mask and - `False` denotes a soft mask. - - .. warning:: Assigning to `_hardmask` does *not* trigger a - hardening or softening of the mask of the - underlying data values. Therefore assigning to - `_hardmask` should only be done in cases when it - is known that the intrinsic mask hardness of the - data values is inconsistent with the - existing value of `_hardmask`. Before assigning - to `_hardmask`, first consider if assigning to - `hardmask`, or calling the `harden_mask` or - `soften_mask` method is a more appropriate course - of action, and use one of those if possible. - - See `hardmask` for details. - - """ - return self._custom["_hardmask"] - - @_hardmask.setter - def _hardmask(self, value): - self._custom["_hardmask"] = value - @property @daskified(_DASKIFIED_VERBOSE) def _axes(self): @@ -4700,11 +4404,8 @@ def Units(self, value): "Consider using the override_units method instead." ) - if not old_units: - self.override_units(value, inplace=True) - return - - if self.Units.equals(value): + if not old_units or self.Units.equals(value): + self._Units = value return dtype = self.dtype @@ -4714,14 +4415,12 @@ def Units(self, value): else: dtype = _dtype_float - def cf_Units(x): - return Units.conform( - x=x, from_units=old_units, to_units=value, inplace=False - ) - dx = self.to_dask_array() - dx = dx.map_blocks(cf_Units, dtype=dtype) - self._set_dask(dx, reset_mask_hardness=False) + dx = dx.map_blocks( + partial(cf_units, from_units=old_units, to_units=value), + dtype=dtype, + ) + self._set_dask(dx) self._Units = value @@ -4791,7 +4490,7 @@ def dtype(self, value): # dask array if dx.dtype != value: dx = dx.astype(value) - self._set_dask(dx, reset_mask_hardness=False) + self._set_dask(dx) @property @daskified(_DASKIFIED_VERBOSE) @@ -4837,15 +4536,21 @@ def hardmask(self): mask, then masked entries may be overwritten with non-missing values. - To allow the unmasking of masked values, the mask must be - softened by setting the `hardmask` attribute to False, or - equivalently with the `soften_mask` method. + .. note:: Setting the `hardmask` attribute does not + immediately change the mask hardness, rather its + value indicates to other methods (such as `where`, + `transpose`, etc.) whether or not the mask needs + hardening or softening prior to an operation being + defined, and those methods will reset the mask + hardness if required. - The mask can be hardened by setting the `hardmask` attribute - to True, or equivalently with the `harden_mask` method. + By contrast, the `harden_mask` and `soften_mask` + methods immediately reset the mask hardness of the + underlying `dask` array, and also set the value of + the `hardmask` attribute. - .. seealso:: `harden_mask`, `soften_mask`, `where`, - `__setitem__` + .. seealso:: `harden_mask`, `soften_mask`, `to_dask_array`, + `where`, `__setitem__` **Examples** @@ -4855,7 +4560,7 @@ def hardmask(self): >>> d[0] = cf.masked >>> print(d.array) [-- 2 3] - >>> d[...]= 999 + >>> d[...] = 999 >>> print(d.array) [-- 999 999] >>> d.hardmask = False @@ -4866,14 +4571,11 @@ def hardmask(self): [-1 -1 -1] """ - return self._hardmask + return self._custom.get("hardmask", _DEFAULT_HARDMASK) @hardmask.setter def hardmask(self, value): - if value: - self.harden_mask() - else: - self.soften_mask() + self._custom["hardmask"] = value @property @daskified(_DASKIFIED_VERBOSE) @@ -4915,26 +4617,6 @@ def is_masked(a): return bool(dx.any()) - @property - @daskified(_DASKIFIED_VERBOSE) - def isscalar(self): - """True if the data is a 0-d scalar array. - - **Examples** - - >>> d = cf.Data(9, 'm') - >>> d.isscalar - True - >>> d = cf.Data([9], 'm') - >>> d.isscalar - False - >>> d = cf.Data([9, 10], 'm') - >>> d.isscalar - False - - """ - return not self.ndim - @property @daskified(_DASKIFIED_VERBOSE) def nbytes(self): @@ -5224,7 +4906,7 @@ def mask(self): dx = self.to_dask_array() mask = da.ma.getmaskarray(dx) - mask_data_obj._set_dask(mask, reset_mask_hardness=False) + mask_data_obj._set_dask(mask) mask_data_obj.override_units(_units_None, inplace=True) mask_data_obj.hardmask = _DEFAULT_HARDMASK @@ -5274,7 +4956,7 @@ def arctan(self, inplace=False): d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() - d._set_dask(da.arctan(dx), reset_mask_hardness=False) + d._set_dask(da.arctan(dx)) d.override_units(_units_radians, inplace=True) @@ -5473,7 +5155,7 @@ def arcsinh(self, inplace=False): d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() - d._set_dask(da.arcsinh(dx), reset_mask_hardness=False) + d._set_dask(da.arcsinh(dx)) d.override_units(_units_radians, inplace=True) @@ -5650,7 +5332,7 @@ def all(self, axis=None, keepdims=True, split_every=None): d = self.copy(array=False) dx = self.to_dask_array() dx = da.all(dx, axis=axis, keepdims=keepdims, split_every=split_every) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) d.hardmask = _DEFAULT_HARDMASK d.override_units(_units_None, inplace=True) return d @@ -5763,7 +5445,7 @@ def any(self, axis=None, keepdims=True, split_every=None): d = self.copy(array=False) dx = self.to_dask_array() dx = da.any(dx, axis=axis, keepdims=keepdims, split_every=split_every) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) d.hardmask = _DEFAULT_HARDMASK d.override_units(_units_None, inplace=True) return d @@ -5943,6 +5625,7 @@ def apply_masking( ) d = _inplace_enabled_define_and_cleanup(self) + dx = self.to_dask_array() mask = None @@ -5967,7 +5650,7 @@ def apply_masking( if mask is not None: dx = da.ma.masked_where(mask, dx) - d._set_dask(dx, reset_mask_hardness=True) + d._set_dask(dx) return d @@ -6002,73 +5685,6 @@ def concatenate_data(cls, data_list, axis): assert len(data_list) == 1 return data_list[0] - @classmethod - def reconstruct_sectioned_data(cls, sections, cyclic=(), hardmask=None): - """Expects a dictionary of Data objects with ordering - information as keys, as output by the section method when called - with a Data object. Returns a reconstructed cf.Data object with - the sections in the original order. - - :Parameters: - - sections: `dict` - The dictionary of `Data` objects with ordering information - as keys. - - :Returns: - - `Data` - The resulting reconstructed Data object. - - **Examples** - - >>> d = cf.Data(numpy.arange(120).reshape(2, 3, 4, 5)) - >>> x = d.section([1, 3]) - >>> len(x) - 8 - >>> e = cf.Data.reconstruct_sectioned_data(x) - >>> e.equals(d) - True - - """ - ndims = len(list(sections.keys())[0]) - - for i in range(ndims - 1, -1, -1): - keys = sorted(sections.keys()) - if i == 0: - if keys[0][i] is None: - assert len(keys) == 1 - return tuple(sections.values())[0] - else: - data_list = [] - for k in keys: - data_list.append(sections[k]) - - out = cls.concatenate_data(data_list, i) - - out.cyclic(cyclic) - if hardmask is not None: - out.hardmask = hardmask - - return out - - if keys[0][i] is not None: - new_sections = {} - new_key = keys[0][:i] - data_list = [] - for k in keys: - if k[:i] == new_key: - data_list.append(sections[k]) - else: - new_sections[new_key] = cls.concatenate_data( - data_list, axis=i - ) - new_key = k[:i] - data_list = [sections[k]] - - new_sections[new_key] = cls.concatenate_data(data_list, i) - sections = new_sections - def argmax(self, axis=None, unravel=False): """Return the indices of the maximum values along an axis. @@ -6149,6 +5765,7 @@ def argmax(self, axis=None, unravel=False): return type(self)(a) + @daskified(_DASKIFIED_VERBOSE) def get_data(self, default=ValueError(), _units=None, _fill_value=None): """Returns the data. @@ -6161,6 +5778,7 @@ def get_data(self, default=ValueError(), _units=None, _fill_value=None): """ return self + @daskified(_DASKIFIED_VERBOSE) def get_units(self, default=ValueError()): """Return the units. @@ -6194,6 +5812,7 @@ def get_units(self, default=ValueError()): except AttributeError: return super().get_units(default=default) + @daskified(_DASKIFIED_VERBOSE) def get_calendar(self, default=ValueError()): """Return the calendar. @@ -6227,6 +5846,7 @@ def get_calendar(self, default=ValueError()): except (AttributeError, KeyError): return super().get_calendar(default=default) + @daskified(_DASKIFIED_VERBOSE) def set_calendar(self, calendar): """Set the calendar. @@ -6256,6 +5876,7 @@ def set_calendar(self, calendar): """ self.Units = Units(self.get_units(default=None), calendar) + @daskified(_DASKIFIED_VERBOSE) def set_units(self, value): """Set the units. @@ -6358,12 +5979,7 @@ def max( @daskified(_DASKIFIED_VERBOSE) @_inplace_enabled(default=False) def maximum_absolute_value( - self, - axes=None, - squeeze=False, - mtol=1, - split_every=None, - inplace=False, + self, axes=None, squeeze=False, mtol=1, split_every=None, inplace=False ): """Calculate maximum absolute values. @@ -6496,12 +6112,7 @@ def min( @daskified(_DASKIFIED_VERBOSE) @_inplace_enabled(default=False) def minimum_absolute_value( - self, - axes=None, - squeeze=False, - mtol=1, - split_every=None, - inplace=False, + self, axes=None, squeeze=False, mtol=1, split_every=None, inplace=False ): """Calculate minimum absolute values. @@ -6983,7 +6594,7 @@ def clip(self, a_min, a_max, units=None, inplace=False, i=False): d = _inplace_enabled_define_and_cleanup(self) dx = self.to_dask_array() dx = da.clip(dx, a_min, a_max) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d @classmethod @@ -7111,7 +6722,7 @@ def compressed(self, inplace=False): meta=np.array((), dtype=dx.dtype), ) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d @daskified(_DASKIFIED_VERBOSE) @@ -7168,7 +6779,7 @@ def cos(self, inplace=False, i=False): d.Units = _units_radians dx = d.to_dask_array() - d._set_dask(da.cos(dx), reset_mask_hardness=False) + d._set_dask(da.cos(dx)) d.override_units(_units_1, inplace=True) @@ -7569,21 +7180,20 @@ def unique(self, split_every=None): """ d = self.copy() - hardmask = d.hardmask - if hardmask: - # Soften a hardmask so that the result doesn't contain a - # seperate missing value for each input chunk that - # contains missing values. For any number greater than 0 - # of missing values in the original data, we only want one - # missing value in the result. - d.soften_mask() + + # Soften the hardmask so that the result doesn't contain a + # seperate missing value for each input chunk that contains + # missing values. For any number greater than 0 of missing + # values in the original data, we only want one missing value + # in the result. + d.soften_mask() dx = d.to_dask_array() dx = Collapse.unique(dx, split_every=split_every) - d._set_dask(dx, reset_mask_hardness=False) - if hardmask: - d.harden_mask() + d._set_dask(dx) + + d.hardmask = _DEFAULT_HARDMASK return d @@ -7827,7 +7437,7 @@ def exp(self, inplace=False, i=False): d.Units = _units_1 dx = d.to_dask_array() - d._set_dask(da.exp(dx), reset_mask_hardness=False) + d._set_dask(da.exp(dx)) return d @@ -7875,7 +7485,7 @@ def insert_dimension(self, position=0, inplace=False): dx = d.to_dask_array() dx = dx.reshape(shape) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) # Expand _axes axis = new_axis_identifier(d._axes) @@ -7885,45 +7495,6 @@ def insert_dimension(self, position=0, inplace=False): return d - @daskified(_DASKIFIED_VERBOSE) - def get_filenames(self): - """Return the names of files containing parts of the data array. - - :Returns: - - `set` - The file names in normalized, absolute form. If the - data is in memory then an empty `set` is returned. - - **Examples** - - >>> f = cf.NetCDFArray(TODODASK) - >>> d = cf.Data(f) - >>> d.get_filenames() - {TODODASK} - - >>> d = cf.Data([1, 2, 3]) - >>> d.get_filenames() - set() - - """ - out = set() - - dx = self.to_dask_array() - hlg = dx.dask - dsk = hlg.to_dict() - for key, value in hlg.get_all_dependencies().items(): - if value: - continue - - # This key has no dependencies, and so is raw data. - a = dsk[key] - if isinstance(a, FileArray): - out.add(abspath(a.get_filename())) - - out.discard(None) - return out - @daskified(_DASKIFIED_VERBOSE) @_deprecated_kwarg_check("size") @_inplace_enabled(default=False) @@ -8226,14 +7797,14 @@ def halo( dx = concatenate([left, dx, right], axis=axis) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) # Special case for tripolar: The northern Y axis halo contains # the values that have been flipped in the X direction. if tripolar: - hardmask = d.hardmask - if hardmask: - d.hardmask = False + # Make sure that we can overwrite any missing values in + # the northern Y axis halo + d.soften_mask() indices1 = indices[:] if fold_index == -1: @@ -8251,10 +7822,10 @@ def halo( dx = d.to_dask_array() dx[tuple(indices1)] = dx[tuple(indices2)] - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) - if hardmask: - d.hardmask = True + # Reset the mask hardness + d.hardmask = self.hardmask # Set expanded axes to be non-cyclic d.cyclic(axes=tuple(depth), iscyclic=False) @@ -8291,8 +7862,8 @@ def harden_mask(self): """ dx = self.to_dask_array() dx = dx.map_blocks(cf_harden_mask, dtype=self.dtype) - self._set_dask(dx, delete_source=False, reset_mask_hardness=False) - self._hardmask = True + self._set_dask(dx, delete_source=False) + self.hardmask = True def has_calendar(self): """Whether a calendar has been set. @@ -8388,8 +7959,8 @@ def soften_mask(self): """ dx = self.to_dask_array() dx = dx.map_blocks(cf_soften_mask, dtype=self.dtype) - self._set_dask(dx, delete_source=False, reset_mask_hardness=False) - self._hardmask = False + self._set_dask(dx, delete_source=False) + self.hardmask = False @daskified(_DASKIFIED_VERBOSE) @_inplace_enabled(default=False) @@ -8443,7 +8014,7 @@ def filled(self, fill_value=None, inplace=False): dx = d.to_dask_array() dx = dx.map_blocks(np.ma.filled, fill_value=fill_value, dtype=d.dtype) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d @@ -8748,7 +8319,7 @@ def flatten(self, axes=None, inplace=False): new_shape.insert(axes[0], reduce(mul, [shape[i] for i in axes], 1)) dx = dx.reshape(new_shape) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d @@ -8783,7 +8354,7 @@ def floor(self, inplace=False, i=False): """ d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() - d._set_dask(da.floor(dx), reset_mask_hardness=False) + d._set_dask(da.floor(dx)) return d @daskified(_DASKIFIED_VERBOSE) @@ -8854,7 +8425,7 @@ def outerproduct(self, a, inplace=False, i=False): dx = d.to_dask_array() dx = da.ufunc.multiply.outer(dx, a) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) d.override_units(d.Units * a.Units, inplace=True) @@ -9017,9 +8588,28 @@ def override_calendar(self, calendar, inplace=False, i=False): d._Units = Units(d.Units._units, calendar) return d - def to_dask_array(self): + def to_dask_array(self, apply_mask_hardness=False): """Convert the data to a `dask` array. + .. warning:: By default, the mask hardness of the returned + dask array might not be the same as that + specified by the `hardmask` attribute. + + This could cause problems if a subsequent + operation on the returned dask array involves the + un-masking of masked values (such as by indexed + assignment). + + To guarantee that the mask hardness of the + returned dassk array is correct, set the + *apply_mask_hardness* parameter to True. + + :Parameters: + + apply_mask_hardness: `bool`, optional + If True then force the mask hardness of the returned + array to be that given by the `hardmask` attribute. + :Returns: `dask.array.Array` @@ -9030,11 +8620,24 @@ def to_dask_array(self): >>> d = cf.Data([1, 2, 3, 4], 'm') >>> dx = d.to_dask_array() >>> dx - >>> dask.array + >>> dask.array >>> dask.array.asanyarray(d) is dx True + >>> d.to_dask_array(apply_mask_hardness=True) + dask.array + + >>> d = cf.Data([1, 2, 3, 4], 'm', hardmask=False) + >>> d.to_dask_array(apply_mask_hardness=True) + dask.array + """ + if apply_mask_hardness: + if self.hardmask: + self.harden_mask() + else: + self.soften_mask() + return self._custom["dask"] @daskified(_DASKIFIED_VERBOSE) @@ -9213,7 +8816,7 @@ def masked_invalid(self, inplace=False): d = _inplace_enabled_define_and_cleanup(self) dx = self.to_dask_array() dx = da.ma.masked_invalid(dx) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d def del_calendar(self, default=ValueError()): @@ -9377,7 +8980,7 @@ def masked_all( ) dx = d.to_dask_array() dx = dx.map_blocks(partial(np.ma.array, mask=True, copy=False)) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d @daskified(_DASKIFIED_VERBOSE) @@ -9507,7 +9110,7 @@ def flip(self, axes=None, inplace=False, i=False): dx = d.to_dask_array() dx = dx[tuple(index)] - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d @@ -9539,19 +9142,20 @@ def inspect(self): inspect(self) + @daskified(_DASKIFIED_VERBOSE) def isclose(self, y, rtol=None, atol=None): - """Return where data are element-wise equal to other, - broadcastable data. + """Return where data are element-wise equal within a tolerance. {{equals tolerance}} - For numeric data arrays, ``d.isclose(y, rtol, atol)`` is - equivalent to ``abs(d - y) <= ``atol + rtol*abs(y)``, otherwise it - is equivalent to ``d == y``. + For numeric data arrays, ``d.isclose(e, rtol, atol)`` is + equivalent to ``abs(d - e) <= atol + rtol*abs(e)``, + otherwise it is equivalent to ``d == e``. :Parameters: y: data_like + The array to compare. atol: `float`, optional The absolute tolerance for all numerical comparisons. By @@ -9564,6 +9168,7 @@ def isclose(self, y, rtol=None, atol=None): :Returns: `bool` + A boolean array of where the data are close to *y*. **Examples** @@ -9587,30 +9192,32 @@ def isclose(self, y, rtol=None, atol=None): [ True True True] """ - if atol is None: - atol = self._atol + a = np.empty((), dtype=self.dtype) + b = np.empty((), dtype=da.asanyarray(y).dtype) + try: + # Check if a numerical isclose is possible + np.isclose(a, b) + except TypeError: + # self and y do not have suitable numeric data types + # (e.g. both are strings) + return self == y + else: + # self and y have suitable numeric data types + if atol is None: + atol = self._atol - if rtol is None: - rtol = self._rtol + if rtol is None: + rtol = self._rtol - units0 = self.Units - units1 = getattr(y, "Units", _units_None) - if units0.isreftime and units1.isreftime: - if not units0.equals(units1): - if not units0.equivalent(units1): - pass + y = conform_units(y, self.Units) - x = self.override_units(_units_1) - y = y.copy() - y.Units = units0 - y.override_units(_units_1, inplace=True) - else: - x = self + dx = da.isclose(self, y, atol=atol, rtol=rtol) - try: - return abs(x - y) <= float(atol) + float(rtol) * abs(y) - except (TypeError, NotImplementedError, IndexError): - return self == y + d = self.copy(array=False) + d._set_dask(dx) + d.hardmask = _DEFAULT_HARDMASK + d.override_units(_units_None, inplace=True) + return d @daskified(_DASKIFIED_VERBOSE) @_inplace_enabled(default=False) @@ -9677,7 +9284,7 @@ def reshape(self, *shape, merge_chunks=True, limit=None, inplace=False): d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() dx = dx.reshape(*shape, merge_chunks=merge_chunks, limit=limit) - d._set_dask(dx, reset_mask_hardness=True) + d._set_dask(dx) return d @daskified(_DASKIFIED_VERBOSE) @@ -9713,7 +9320,7 @@ def rint(self, inplace=False, i=False): """ d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() - d._set_dask(da.rint(dx), reset_mask_hardness=False) + d._set_dask(da.rint(dx)) return d @daskified(_DASKIFIED_VERBOSE) @@ -9838,7 +9445,7 @@ def round(self, decimals=0, inplace=False, i=False): """ d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() - d._set_dask(da.round(dx, decimals=decimals), reset_mask_hardness=False) + d._set_dask(da.round(dx, decimals=decimals)) return d def stats( @@ -10075,33 +9682,53 @@ def swapaxes(self, axis0, axis1, inplace=False, i=False): d = _inplace_enabled_define_and_cleanup(self) dx = self.to_dask_array() dx = da.swapaxes(dx, axis0, axis1) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d - def fits_in_memory(self, itemsize): - """Return True if the master array is small enough to be - retained in memory. + def fits_in_memory(self): + """Return True if the array is small enough to be retained in + memory. + + Returns True if the size of the array with all delayed + operations computed, always including space for a full boolean + mask, is small enough to be retained in available memory. + + .. note:: The delayed operations are actually not computed by + `fits_in_memory`, so it is possible that an + intermediate operation may require more than the + available memory, even if the final array does not. + + .. seealso:: `array`, `compute`, `nbytes`, `persist`, + `cf.free_memory` :Parameters: - itemsize: `int` + itemsize: deprecated at version TODODASK The number of bytes per word of the master data array. :Returns: `bool` + Whether or not the computed array fits in memory. **Examples** - >>> print(d.fits_in_memory(8)) + >>> d = cf.Data([1], 'm') + >>> d.fits_in_memory() + True + + Create a double precision (8 bytes per word) array that is + approximately twice the size of the available memory: + + >>> size = int(2 * cf.free_memory() / 8) + >>> d = cf.Data.empty((size,), dtype=float) + >>> d.fits_in_memory() False + >>> d.nbytes * (1 + 1/8) > cf.free_memory() + True """ - # ------------------------------------------------------------ - # Note that self._size*(itemsize+1) is the array size in bytes - # including space for a full boolean mask - # ------------------------------------------------------------ - return self.size * (itemsize + 1) <= free_memory() - fm_threshold() + return self.size * (self.dtype.itemsize + 1) <= free_memory() @_deprecated_kwarg_check("i") @_inplace_enabled(default=False) @@ -10306,8 +9933,11 @@ def where( """ d = _inplace_enabled_define_and_cleanup(self) + # Missing values could be affected, so make sure that the mask + # hardness has been applied. + dx = d.to_dask_array(apply_mask_hardness=True) + units = d.Units - dx = d.to_dask_array() # Parse condition if getattr(condition, "isquery", False): @@ -10370,10 +10000,6 @@ def where( ) d._set_dask(dx) - # Note: No need to run `_reset_mask_hardness` at this point - # because the mask hardness has already been correctly - # set in `cf_where`. - return d @daskified(_DASKIFIED_VERBOSE) @@ -10430,7 +10056,7 @@ def sin(self, inplace=False, i=False): d.Units = _units_radians dx = d.to_dask_array() - d._set_dask(da.sin(dx), reset_mask_hardness=False) + d._set_dask(da.sin(dx)) d.override_units(_units_1, inplace=True) @@ -10491,7 +10117,7 @@ def sinh(self, inplace=False): d.Units = _units_radians dx = d.to_dask_array() - d._set_dask(da.sinh(dx), reset_mask_hardness=False) + d._set_dask(da.sinh(dx)) d.override_units(_units_1, inplace=True) @@ -10550,7 +10176,7 @@ def cosh(self, inplace=False): d.Units = _units_radians dx = d.to_dask_array() - d._set_dask(da.cosh(dx), reset_mask_hardness=False) + d._set_dask(da.cosh(dx)) d.override_units(_units_1, inplace=True) @@ -10612,7 +10238,7 @@ def tanh(self, inplace=False): d.Units = _units_radians dx = d.to_dask_array() - d._set_dask(da.tanh(dx), reset_mask_hardness=False) + d._set_dask(da.tanh(dx)) d.override_units(_units_1, inplace=True) @@ -10650,7 +10276,7 @@ def log(self, base=None, inplace=False, i=False): dx = da.log(dx) dx /= da.log(base) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) d.override_units( _units_1, inplace=True @@ -10747,7 +10373,6 @@ def squeeze(self, axes=None, inplace=False, i=False): f"Can't squeeze {d.__class__.__name__}: " f"Can't remove axis of size {shape[i]}" ) - # --- End: if if not axes: return d @@ -10756,7 +10381,7 @@ def squeeze(self, axes=None, inplace=False, i=False): # one size 1 axis needs squeezing. dx = d.to_dask_array() dx = dx.squeeze(axis=tuple(axes)) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) # Remove the squeezed axes names d._axes = [axis for i, axis in enumerate(d._axes) if i not in axes] @@ -10819,7 +10444,7 @@ def tan(self, inplace=False, i=False): d.Units = _units_radians dx = d.to_dask_array() - d._set_dask(da.tan(dx), reset_mask_hardness=False) + d._set_dask(da.tan(dx)) d.override_units(_units_1, inplace=True) @@ -10872,7 +10497,7 @@ def to_memory(self): "'Data.to_memory' is not available. " "Consider using 'Data.persist' instead." ) - + @daskified(_DASKIFIED_VERBOSE) @_deprecated_kwarg_check("i") @_inplace_enabled(default=False) @@ -10935,7 +10560,7 @@ def transpose(self, axes=None, inplace=False, i=False): raise ValueError( f"Can't transpose: Axes don't match array: {axes}" ) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d @@ -10974,7 +10599,7 @@ def trunc(self, inplace=False, i=False): """ d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() - d._set_dask(da.trunc(dx), reset_mask_hardness=False) + d._set_dask(da.trunc(dx)) return d @classmethod @@ -11279,6 +10904,7 @@ def func( """ d = _inplace_enabled_define_and_cleanup(self) + dx = d.to_dask_array() # TODODASK: Steps to preserve invalid values shown, taking same @@ -11299,7 +10925,7 @@ def func( # Step 3: reattach original mask onto the output data dx = da.ma.masked_array(dx, mask=dx_mask) - d._set_dask(dx, reset_mask_hardness=True) + d._set_dask(dx) if units is not None: d.override_units(units, inplace=True) @@ -11423,7 +11049,7 @@ def roll(self, axis, shift, inplace=False, i=False): dx = d.to_dask_array() dx = da.roll(dx, shift, axis=axis) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) return d @@ -11778,7 +11404,7 @@ def sum_of_weights2( if not units: units = _units_None else: - units = units ** 2 + units = units**2 d.override_units(units, inplace=True) @@ -11961,7 +11587,7 @@ def var( units = d.Units if units: - d.override_units(units ** 2, inplace=True) + d.override_units(units**2, inplace=True) return d @@ -12095,11 +11721,11 @@ def square(self, dtype=None, inplace=False): d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() dx = da.square(dx, dtype=dtype) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) units = d.Units if units: - d.override_units(units ** 2, inplace=True) + d.override_units(units**2, inplace=True) return d @@ -12165,12 +11791,12 @@ def sqrt(self, dtype=None, inplace=False): d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() dx = da.sqrt(dx, dtype=dtype) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) units = d.Units if units: try: - d.override_units(units ** 0.5, inplace=True) + d.override_units(units**0.5, inplace=True) except ValueError as e: raise type(e)( f"Incompatible units for taking a square root: {units!r}" @@ -12547,7 +12173,7 @@ def _collapse( dx = d.to_dask_array() dx = func(dx, **kwargs) - d._set_dask(dx, reset_mask_hardness=True) + d._set_dask(dx) return d, weights diff --git a/cf/data/mixin/deprecations.py b/cf/data/mixin/deprecations.py index 9a9914cf67..78525031e8 100644 --- a/cf/data/mixin/deprecations.py +++ b/cf/data/mixin/deprecations.py @@ -109,10 +109,7 @@ def Data(self): def dtvarray(self): """Deprecated at version 3.0.0.""" _DEPRECATION_ERROR_ATTRIBUTE( - self, - "dtvarray", - version="3.0.0", - removed_at="4.0.0", + self, "dtvarray", version="3.0.0", removed_at="4.0.0" ) # pragma: no cover @property @@ -123,10 +120,7 @@ def in_memory(self): """ _DEPRECATION_ERROR_ATTRIBUTE( - self, - "in_memory", - version="TODODASK", - removed_at="5.0.0", + self, "in_memory", version="TODODASK", removed_at="5.0.0" ) # pragma: no cover @property @@ -154,6 +148,33 @@ def ismasked(self): removed_at="5.0.0", ) # pragma: no cover + @property + def isscalar(self): + """True if the data is a 0-d scalar array. + + Deprecated at version TODODASK. Use `d.ndim == 0`` instead. + + **Examples** + + >>> d = cf.Data(9, 'm') + >>> d.isscalar + True + >>> d = cf.Data([9], 'm') + >>> d.isscalar + False + >>> d = cf.Data([9, 10], 'm') + >>> d.isscalar + False + + """ + _DEPRECATION_ERROR_ATTRIBUTE( + self, + "isscalar", + message="Use 'd.ndim == 0' instead", + version="TODODASK", + removed_at="5.0.0", + ) # pragma: no cover + @property def ispartitioned(self): """True if the data array is partitioned. @@ -235,7 +256,8 @@ def expand_dims(self, position=0, i=False): ) # pragma: no cover def files(self): - """Deprecated at version 3.4.0, use method `get_filenames` instead.""" + """Deprecated at version 3.4.0, use method `get_filenames` + instead.""" _DEPRECATION_ERROR_METHOD( self, "files", @@ -287,6 +309,23 @@ def close(self): removed_at="5.0.0", ) # pragma: no cover + def get_filenames(self): + """Return the names of files containing parts of the data array. + + Deprecated at version TODODASK. + + :Returns: + + `set` + The file names in normalized, absolute form. If the + data is in memory then an empty `set` is returned. + + """ + raise DeprecationError( + "Data method 'get_filenames' has been deprecated at " + "version TODODASK and is not available." + ) # pragma: no cover + def chunk(self, chunksize=None, total=None, omit_axes=None, pmshape=None): """Partition the data array. @@ -715,9 +754,7 @@ def partition_boundaries(self): def save_to_disk(self, itemsize=None): """Deprecated.""" _DEPRECATION_ERROR_METHOD( - self, - "save_to_disk", - removed_at="4.0.0", + self, "save_to_disk", removed_at="4.0.0" ) # pragma: no cover def to_disk(self): @@ -738,12 +775,9 @@ def to_disk(self): """ _DEPRECATION_ERROR_METHOD( - self, - "to_disk", - version="TODODASK", - removed_at="5.0.0", + self, "to_disk", version="TODODASK", removed_at="5.0.0" ) # pragma: no cover - + @staticmethod def seterr(all=None, divide=None, over=None, under=None, invalid=None): """Set how floating-point errors in the results of arithmetic @@ -894,3 +928,39 @@ def seterr(all=None, divide=None, over=None, under=None, invalid=None): "manipulations. This may change in the future (see " "https://github.com/dask/dask/issues/3245 for more details)." ) + + @classmethod + def reconstruct_sectioned_data(cls, sections, cyclic=(), hardmask=None): + """Expects a dictionary of Data objects with ordering + information as keys, as output by the section method when called + with a Data object. Returns a reconstructed cf.Data object with + the sections in the original order. + + Deprecated at version TODODASK and is no longer available. + + :Parameters: + + sections: `dict` + The dictionary of `Data` objects with ordering information + as keys. + + :Returns: + + `Data` + The resulting reconstructed Data object. + + **Examples** + + >>> d = cf.Data(numpy.arange(120).reshape(2, 3, 4, 5)) + >>> x = d.section([1, 3]) + >>> len(x) + 8 + >>> e = cf.Data.reconstruct_sectioned_data(x) + >>> e.equals(d) + True + + """ + raise DeprecationError( + "Data method 'reconstruct_sectioned_data' has been deprecated " + "at version TODODASK and is no longer available" + ) diff --git a/cf/data/utils.py b/cf/data/utils.py index 96dc58bbd3..249dcbfd52 100644 --- a/cf/data/utils.py +++ b/cf/data/utils.py @@ -16,6 +16,8 @@ from ..units import Units from .dask_utils import cf_YMDhms +_units_None = Units(None) + def _is_numeric_dtype(array): """True if the given array is of a numeric or boolean data type. @@ -577,39 +579,39 @@ def conform_units(value, units): **Examples** - >>> cf.data.utils.conform_units(1, cf.Units('metres')) + >>> cf.data.utils.conform_units(1, cf.Units('m')) 1 - >>> cf.data.utils.conform_units([1, 2, 3], cf.Units('metres')) + >>> cf.data.utils.conform_units([1, 2, 3], cf.Units('m')) [1, 2, 3] >>> import numpy as np - >>> cf.data.utils.conform_units(np.array([1, 2, 3]), cf.Units('metres')) + >>> cf.data.utils.conform_units(np.array([1, 2, 3]), cf.Units('m')) array([1, 2, 3]) - >>> cf.data.utils.conform_units('string', cf.Units('metres')) + >>> cf.data.utils.conform_units('string', cf.Units('m')) 'string' >>> d = cf.Data([1, 2] , 'm') - >>> cf.data.utils.conform_units(d, cf.Units('metres')) + >>> cf.data.utils.conform_units(d, cf.Units('m')) >>> d = cf.Data([1, 2] , 'km') - >>> cf.data.utils.conform_units(d, cf.Units('metres')) - + >>> cf.data.utils.conform_units(d, cf.Units('m')) + >>> cf.data.utils.conform_units(d, cf.Units('s')) + Traceback (most recent call last): ... ValueError: Units are incompatible with units """ - try: - value_units = value.Units - except AttributeError: - pass - else: - if value_units.equivalent(units): - if value_units != units: - value = value.copy() - value.Units = units - elif value_units and units: - raise ValueError( - f"Units {value_units!r} are incompatible with units {units!r}" - ) + value_units = getattr(value, "Units", None) + if value_units is None: + return value + + if value_units.equivalent(units): + if value_units != units: + value = value.copy() + value.Units = units + elif value_units and units: + raise ValueError( + f"Units {value_units!r} are incompatible with units {units!r}" + ) return value @@ -654,6 +656,6 @@ def YMDhms(d, attr): d = d._asdatetime() dx = d.to_dask_array() dx = dx.map_blocks(partial(cf_YMDhms, attr=attr), dtype=int) - d._set_dask(dx, reset_mask_hardness=False) + d._set_dask(dx) d.override_units(Units(None), inplace=True) return d diff --git a/cf/formula_terms.py b/cf/formula_terms.py index bf1bdd1214..142ad22bf5 100644 --- a/cf/formula_terms.py +++ b/cf/formula_terms.py @@ -2088,10 +2088,8 @@ def formula( then a `None` is returned for all of the tuple elements. """ - standard_name = ( - coordinate_reference.coordinate_conversion.get_parameter( - "standard_name", None - ) + standard_name = coordinate_reference.coordinate_conversion.get_parameter( + "standard_name", None ) if standard_name is not None: diff --git a/cf/functions.py b/cf/functions.py index 8da0b6d8b6..71e5087776 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -29,7 +29,6 @@ from dask.utils import parse_bytes from numpy import all as _numpy_all from numpy import allclose as _x_numpy_allclose -from numpy import isclose as _x_numpy_isclose from numpy import shape as _numpy_shape from numpy import take as _numpy_take from numpy import tile as _numpy_tile @@ -1829,39 +1828,6 @@ def _numpy_allclose(a, b, rtol=None, atol=None, verbose=None): return out -def _numpy_isclose(a, b, rtol=None, atol=None): - """Returns a boolean array where two broadcastable arrays are - element-wise equal within a tolerance. - - The tolerance values are positive, typically very small numbers. The - relative difference (``rtol * abs(b)``) and the absolute difference - ``atol`` are added together to compare against the absolute difference - between ``a`` and ``b``. - - :Parameters: - - a, b: array_like - Input arrays to compare. - - atol: `float`, optional - The absolute tolerance for all numerical comparisons, By - default the value returned by the `atol` function is used. - - rtol: `float`, optional - The relative tolerance for all numerical comparisons, By - default the value returned by the `rtol` function is used. - - :Returns: - - `numpy.ndarray` - - """ - try: - return _x_numpy_isclose(a, b, rtol=rtol, atol=atol) - except (IndexError, NotImplementedError, TypeError): - return a == b - - def parse_indices(shape, indices, cyclic=False, keepdims=True): """Parse indices for array access and assignment. @@ -3108,12 +3074,7 @@ def _DEPRECATION_ERROR(message="", version="3.0.0"): def _DEPRECATION_ERROR_ARG( - instance, - method, - arg, - message="", - version="3.0.0", - removed_at="4.0.0", + instance, method, arg, message="", version="3.0.0", removed_at="4.0.0" ): if removed_at: removed_at = f" and will be removed at version {removed_at}" @@ -3228,11 +3189,10 @@ def _DEPRECATION_ERROR_ATTRIBUTE( ): if removed_at: removed_at = f" and will be removed at version {removed_at}" - + raise DeprecationError( f"{instance.__class__.__name__} attribute {attribute!r} has been " - f"deprecated at version {version} and will be removed at version " - f"{removed_at}. {message}" + f"deprecated at version {version}{removed_at}. {message}" ) diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index 3c5df8b758..ec9f4804a5 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -110,9 +110,7 @@ def setUp(self): ] for expected_warning in expexted_warning_msgs: warnings.filterwarnings( - "ignore", - category=RuntimeWarning, - message=expected_warning, + "ignore", category=RuntimeWarning, message=expected_warning ) def test_Data_equals(self): @@ -311,21 +309,13 @@ def test_Data_equals(self): ) # ...including masked string arrays sa4 = cf.Data( - np.ma.array( - ["one", "two", "three"], - mask=[0, 0, 1], - dtype="S5", - ), + np.ma.array(["one", "two", "three"], mask=[0, 0, 1], dtype="S5"), "m", chunks=mask_test_chunksize, ) self.assertTrue(sa4.equals(sa4.copy())) sa5 = cf.Data( - np.ma.array( - ["one", "two", "three"], - mask=[0, 1, 0], - dtype="S5", - ), + np.ma.array(["one", "two", "three"], mask=[0, 1, 0], dtype="S5"), "m", chunks=mask_test_chunksize, ) @@ -1997,7 +1987,6 @@ def test_Data_year_month_day_hour_minute_second(self): with self.assertRaises(ValueError): cf.Data([[1, 2]], units="m").year - @unittest.skipIf(TEST_DASKIFIED_ONLY, "'NoneType' is not iterable") def test_Data_BINARY_AND_UNARY_OPERATORS(self): if self.test_only and inspect.stack()[0][3] not in self.test_only: return @@ -2053,12 +2042,12 @@ def test_Data_BINARY_AND_UNARY_OPERATORS(self): ) try: - d ** x + d**x except Exception: pass else: message = "Failed in {!r}**{!r}".format(d, x) - self.assertTrue((d ** x).all(), message) + self.assertTrue((d**x).all(), message) # --- End: for for a0 in arrays: @@ -2084,10 +2073,14 @@ def test_Data_BINARY_AND_UNARY_OPERATORS(self): self.assertTrue( (d // x).equals(cf.Data(a0 // x, "m"), verbose=1), message ) - message = "Failed in {!r}**{}".format(d, x) - self.assertTrue( - (d ** x).equals(cf.Data(a0 ** x, "m2"), verbose=1), message - ) + # TODODASK SB: re-instate this once _combined_units is sorted, + # presently fails with error: + # AttributeError: 'Data' object has no attribute '_size' + # + # message = "Failed in {!r}**{}".format(d, x) + # self.assertTrue( + # (d ** x).equals(cf.Data(a0 ** x, "m2"), verbose=1), message + # ) message = "Failed in {!r}.__truediv__{}".format(d, x) self.assertTrue( d.__truediv__(x).equals( @@ -2128,12 +2121,12 @@ def test_Data_BINARY_AND_UNARY_OPERATORS(self): ) try: - x ** d + x**d except Exception: pass else: message = "Failed in {}**{!r}".format(x, d) - self.assertTrue((x ** d).all(), message) + self.assertTrue((x**d).all(), message) a = a0.copy() try: @@ -2200,18 +2193,21 @@ def test_Data_BINARY_AND_UNARY_OPERATORS(self): e.equals(cf.Data(a, "m"), verbose=1), message ) - a = a0.copy() - try: - a **= x - except TypeError: - pass - else: - e = d.copy() - e **= x - message = "Failed in {!r}**={}".format(d, x) - self.assertTrue( - e.equals(cf.Data(a, "m2"), verbose=1), message - ) + # TODODASK SB: re-instate this once _combined_units is sorted, + # presently fails with error, as with __pow__: + # AttributeError: 'Data' object has no attribute '_size' + # a = a0.copy() + # try: + # a **= x + # except TypeError: + # pass + # else: + # e = d.copy() + # e **= x + # message = "Failed in {!r}**={}".format(d, x) + # self.assertTrue( + # e.equals(cf.Data(a, "m2"), verbose=1), message + # ) a = a0.copy() try: @@ -2245,12 +2241,12 @@ def test_Data_BINARY_AND_UNARY_OPERATORS(self): ) try: - d ** x + d**x except Exception: pass else: self.assertTrue( - (x ** d).all(), "{}**{}".format(x, repr(d)) + (x**d).all(), "{}**{}".format(x, repr(d)) ) self.assertTrue( @@ -2493,14 +2489,6 @@ def test_Data_section(self): self.assertEqual(key, (None, None, None)) self.assertTrue(value.equals(d)) - @unittest.skipIf(TEST_DASKIFIED_ONLY, "Needs reconstruct_sectioned_data") - def test_Data_reconstruct_sectioned_data(self): - if self.test_only and inspect.stack()[0][3] not in self.test_only: - return - - # TODODASK: Write when Data.reconstruct_sectioned_data is - # daskified - @unittest.skipIf(TEST_DASKIFIED_ONLY, "no attr. 'partition_configuration'") def test_Data_count(self): if self.test_only and inspect.stack()[0][3] not in self.test_only: @@ -2532,7 +2520,6 @@ def test_Data_exp(self): # self.assertTrue((d.array==c).all()) so need a # check which accounts for floating point calcs: np.testing.assert_allclose(d.array, c) - # --- End: for d = cf.Data(a, "m") with self.assertRaises(Exception): @@ -2542,7 +2529,7 @@ def test_Data_func(self): if self.test_only and inspect.stack()[0][3] not in self.test_only: return - a = np.array([[np.e, np.e ** 2, np.e ** 3.5], [0, 1, np.e ** -1]]) + a = np.array([[np.e, np.e**2, np.e**3.5], [0, 1, np.e**-1]]) # Using sine as an example function to apply b = np.sin(a) @@ -2590,7 +2577,7 @@ def test_Data_log(self): return # Test natural log, base e - a = np.array([[np.e, np.e ** 2, np.e ** 3.5], [0, 1, np.e ** -1]]) + a = np.array([[np.e, np.e**2, np.e**3.5], [0, 1, np.e**-1]]) b = np.log(a) c = cf.Data(a, "s") d = c.log() @@ -2604,7 +2591,7 @@ def test_Data_log(self): self.assertEqual(c.shape, b.shape) # Test another base, using 10 as an example (special managed case) - a = np.array([[10, 100, 10 ** 3.5], [0, 1, 0.1]]) + a = np.array([[10, 100, 10**3.5], [0, 1, 0.1]]) b = np.log10(a) c = cf.Data(a, "s") d = c.log(base=10) @@ -2612,7 +2599,7 @@ def test_Data_log(self): self.assertEqual(d.shape, b.shape) # Test an arbitrary base, using 4 (not a special managed case like 10) - a = np.array([[4, 16, 4 ** 3.5], [0, 1, 0.25]]) + a = np.array([[4, 16, 4**3.5], [0, 1, 0.25]]) b = np.log(a) / np.log(4) # the numpy way, using log rules from school c = cf.Data(a, "s") d = c.log(base=4) @@ -2866,7 +2853,7 @@ def test_Data_where(self): (e.array == [[-999, -999, -999], [5, -999, -999], [6, 7, 8]]).all() ) - d.soften_mask() + d.hardmask = False e = d.where(a > 5, None, -999) self.assertTrue(e.shape == d.shape) self.assertTrue((e.array.mask == False).all()) @@ -2883,9 +2870,6 @@ def test_Data_where(self): self.assertTrue((e.array == a).all()) def test_Data__init__compression(self): - if self.test_only and inspect.stack()[0][3] not in self.test_only: - return - import cfdm # Ragged @@ -3641,32 +3625,23 @@ def test_Data_collapse_units(self): ): self.assertEqual(func().Units, d.Units) - for func in ( - d.sum_of_squares, - d.var, - ): - self.assertEqual(func().Units, d.Units ** 2) + for func in (d.sum_of_squares, d.var): + self.assertEqual(func().Units, d.Units**2) - for func in ( - d.sum_of_weights, - d.sum_of_weights2, - ): + for func in (d.sum_of_weights, d.sum_of_weights2): self.assertEqual(func().Units, cf.Units()) # Weighted w = cf.Data(1, "m") self.assertEqual(d.integral(weights=w).Units, d.Units * w.Units) self.assertEqual(d.sum_of_weights(weights=w).Units, w.Units) - self.assertEqual(d.sum_of_weights2(weights=w).Units, w.Units ** 2) + self.assertEqual(d.sum_of_weights2(weights=w).Units, w.Units**2) # Dimensionless data d = cf.Data([1, 2]) self.assertEqual(d.integral(weights=w).Units, w.Units) - for func in ( - d.sum_of_squares, - d.var, - ): + for func in (d.sum_of_squares, d.var): self.assertEqual(func().Units, cf.Units()) # TODODASK - add in mean_of_upper_decile when it's daskified @@ -3727,11 +3702,7 @@ def test_Data_collapse_dtype(self): # Cases for which both d and e collapse to a result of the # double of same data type for x, r in zip((d, e), ("i8", "f8")): - for func in ( - x.integral, - x.sum, - x.sum_of_squares, - ): + for func in (x.integral, x.sum, x.sum_of_squares): self.assertEqual(func().dtype, r) # Cases for which both d and e collapse to a result of double @@ -3749,10 +3720,7 @@ def test_Data_collapse_dtype(self): self.assertEqual(func().dtype, r) x = d - for func in ( - x.sum_of_weights, - x.sum_of_weights2, - ): + for func in (x.sum_of_weights, x.sum_of_weights2): self.assertEqual(func().dtype, "i8") # Weights @@ -3808,11 +3776,19 @@ def test_Data_set_units(self): d.set_units("km") def test_Data_allclose(self): + d = cf.Data(1, "m") + for x in (1, [1], np.array([[1]]), da.from_array(1), cf.Data(1)): + self.assertTrue(d.allclose(x).array) + d = cf.Data([1000, 2500], "metre") e = cf.Data([1, 2.5], "km") self.assertTrue(d.allclose(e)) + e = cf.Data([1, 999], "km") + self.assertFalse(d.allclose(e)) + d = cf.Data([[1000, 2500], [1000, 2500]], "metre") + e = cf.Data([1, 2.5], "km") self.assertTrue(d.allclose(e)) d = cf.Data(["ab", "cdef"]) @@ -3831,6 +3807,44 @@ def test_Data_allclose(self): with self.assertRaises(ValueError): d.allclose([1, 2]) + # Incompatible units + d = cf.Data([[1000, 2500]], "m") + e = cf.Data([1, 2.5], "s") + with self.assertRaises(ValueError): + d.allclose(e) + + def test_Data_isclose(self): + d = cf.Data(1, "m") + for x in (1, [1], np.array([[1]]), da.from_array(1), cf.Data(1)): + self.assertTrue(d.isclose(x).array) + + self.assertFalse(d.isclose(1.1)) + + d = cf.Data([[1000, 2500]], "m") + e = cf.Data([1, 2.5], "km") + f = d.isclose(e) + self.assertEqual(f.shape, d.shape) + self.assertTrue((f.array == True).all()) + + e = cf.Data([99, 99], "km") + f = d.isclose(e) + self.assertEqual(f.shape, d.shape) + self.assertTrue((f.array == False).all()) + + d = cf.Data(1, "days since 2000-01-01") + e = cf.Data(0, "days since 2000-01-02") + self.assertTrue(d.isclose(e).array) + + # Strings + d = cf.Data(["foo", "bar"]) + self.assertTrue((d.isclose(["foo", "bar"]).array == True).all()) + + # Incompatible units + d = cf.Data([[1000, 2500]], "m") + e = cf.Data([1, 2.5], "s") + with self.assertRaises(ValueError): + d.isclose(e) + def test_Data_to_dask_array(self): d = cf.Data([1, 2, 3, 4], "m") d.Units = cf.Units("km") @@ -3846,10 +3860,6 @@ def test_Data_flat(self): list(d.flat(ignore_masked=False)), [1, np.ma.masked, 3, 4] ) - @unittest.skipIf(TEST_DASKIFIED_ONLY, "Needs updated NetCDFArray to test") - def test_Data_get_filenames(self): - pass - def test_Data_tolist(self): for x in (1, [1, 2], [[1, 2], [3, 4]]): d = cf.Data(x) @@ -3942,7 +3952,7 @@ def test_Data_masked_all(self): a = np.ma.masked_all((), dtype=dtype) d = cf.Data.masked_all((), dtype=dtype) self.assertEqual(d.dtype, a.dtype) - + def test_Data_atol(self): d = cf.Data(1) self.assertEqual(d._atol, cf.atol()) @@ -3954,15 +3964,147 @@ def test_Data_rtol(self): self.assertEqual(d._rtol, cf.rtol()) cf.rtol(0.001) self.assertEqual(d._rtol, 0.001) - + + def test_Data_hardmask(self): + d = cf.Data([1, 2, 3]) + d.hardmask = True + self.assertTrue(d.hardmask) + self.assertEqual(len(d.to_dask_array().dask.layers), 1) + + d[0] = cf.masked + self.assertTrue((d.array.mask == [True, False, False]).all()) + d[...] = 999 + self.assertTrue((d.array.mask == [True, False, False]).all()) + d.hardmask = False + self.assertFalse(d.hardmask) + d[...] = -1 + self.assertTrue((d.array.mask == [False, False, False]).all()) + + def test_Data_harden_mask(self): + d = cf.Data([1, 2, 3], hardmask=False) + d.harden_mask() + self.assertTrue(d.hardmask) + self.assertEqual(len(d.to_dask_array().dask.layers), 2) + + def test_Data_soften_mask(self): + d = cf.Data([1, 2, 3], hardmask=True) + d.soften_mask() + self.assertFalse(d.hardmask) + self.assertEqual(len(d.to_dask_array().dask.layers), 2) + + def test_Data_compressed_array(self): + import cfdm + + f = cfdm.read("DSG_timeSeries_contiguous.nc")[0] + f = f.data + d = cf.Data(cf.RaggedContiguousArray(source=f.source())) + self.assertTrue((d.compressed_array == f.compressed_array).all()) + + d = cf.Data([1, 2, 3], "m") + with self.assertRaises(Exception): + d.compressed_array + + # TODO: when cfdm>1.9.0.3 is released (i.e. a release that + # includes https://github.com/NCAS-CMS/cfdm/pull/184), + # we can replace the loose "(Exception)" with the tight + # "(ValueError)" + def test_Data_inspect(self): d = cf.Data([9], "m") - + f = io.StringIO() with contextlib.redirect_stdout(f): self.assertIsNone(d.inspect()) - - + + def test_Data_fits_in_memory(self): + size = int(0.1 * cf.free_memory() / 8) + d = cf.Data.empty((size,), dtype=float) + self.assertTrue(d.fits_in_memory()) + + size = int(2 * cf.free_memory() / 8) + d = cf.Data.empty((size,), dtype=float) + self.assertFalse(d.fits_in_memory()) + + def test_Data_get_compressed(self): + import cfdm + + # Compressed + f = cfdm.read("DSG_timeSeries_contiguous.nc")[0] + f = f.data + d = cf.Data(cf.RaggedContiguousArray(source=f.source())) + + self.assertEqual(d.get_compressed_axes(), f.get_compressed_axes()) + self.assertEqual(d.get_compression_type(), f.get_compression_type()) + self.assertEqual( + d.get_compressed_dimension(), f.get_compressed_dimension() + ) + + # Uncompressed + d = cf.Data(9) + + self.assertEqual(d.get_compressed_axes(), []) + self.assertEqual(d.get_compression_type(), "") + + with self.assertRaises(ValueError): + d.get_compressed_dimension() + + def test_Data_Units(self): + d = cf.Data(100, "m") + self.assertEqual(d.Units, cf.Units("m")) + + d.Units = cf.Units("km") + self.assertEqual(d.Units, cf.Units("km")) + self.assertEqual(d.array, 0.1) + + # Assign non-equivalent units + with self.assertRaises(ValueError): + d.Units = cf.Units("watt") + + # Delete units + with self.assertRaises(ValueError): + del d.Units + + def test_Data_get_data(self): + d = cf.Data(9) + self.assertIs(d, d.get_data()) + + def test_Data_get_count(self): + import cfdm + + f = cfdm.read("DSG_timeSeries_contiguous.nc")[0] + f = f.data + d = cf.Data(cf.RaggedContiguousArray(source=f.source())) + self.assertIsInstance(d.get_count(), cfdm.Count) + + d = cf.Data(9, "m") + with self.assertRaises(ValueError): + d.get_count() + + def test_Data_get_index(self): + import cfdm + + f = cfdm.read("DSG_timeSeries_indexed.nc")[0] + f = f.data + d = cf.Data(cf.RaggedIndexedArray(source=f.source())) + self.assertIsInstance(d.get_index(), cfdm.Index) + + d = cf.Data(9, "m") + with self.assertRaises(ValueError): + d.get_index() + + def test_Data_get_list(self): + import cfdm + + f = cfdm.read("gathered.nc")[0] + f = f.data + d = cf.Data(cf.GatheredArray(source=f.source())) + self.assertIsInstance(d.get_list(), cfdm.List) + + d = cf.Data(9, "m") + with self.assertRaises(ValueError): + d.get_list() + + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) cf.environment() diff --git a/cf/test/test_Data_utils.py b/cf/test/test_Data_utils.py index 78689b7b76..93fbc09e22 100644 --- a/cf/test/test_Data_utils.py +++ b/cf/test/test_Data_utils.py @@ -271,6 +271,19 @@ def test_Data_Utils_first_non_missing_value(self): with self.assertRaises(ValueError): cf.data.utils.first_non_missing_value(d, method="bad") + def test_Data_Utils_conform_units(self): + for x in (1, [1, 2], "foo", np.array([[1]])): + self.assertEqual(cf.data.utils.conform_units(x, cf.Units("m")), x) + + d = cf.Data([1000, 2000], "m") + e = cf.data.utils.conform_units(d, cf.Units("m")) + self.assertIs(e, d) + e = cf.data.utils.conform_units(d, cf.Units("km")) + self.assertTrue(e.equals(cf.Data([1, 2], "km"), ignore_data_type=True)) + + with self.assertRaises(ValueError): + cf.data.utils.conform_units(d, cf.Units("s")) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/requirements.txt b/requirements.txt index 49aa387a59..32abad3d00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ numpy>=1.22 cfdm>=1.9.1.0, <1.9.2.0 psutil>=0.6.0 cfunits>=3.3.4 -dask>=2022.03.0 +dask>=2022.6.0