Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dask: Data.isclose #411

Merged
merged 12 commits into from
Jun 21, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ repos:
# Use specific format-enforcing pre-commit hooks from the core library
# with the default configuration (see pre-commit.com for documentation)
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.4.0
rev: v4.3.0
hooks:
- id: check-ast
- id: debug-statements
Expand All @@ -23,7 +23,7 @@ repos:
# (see https://black.readthedocs.io/en/stable/ for documentation and see
# the cf-python pyproject.toml file for our custom black configuration)
- repo: https://github.com/ambv/black
rev: 21.5b0
rev: 22.3.0
hooks:
- id: black
language_version: python3
Expand All @@ -48,7 +48,7 @@ repos:
# (see https://flake8.pycqa.org/en/latest/ for documentation and see
# the cf-python .flake8 file for our custom flake8 configuration)
- repo: https://gitlab.com/pycqa/flake8
rev: 3.9.1
rev: 3.9.2
hooks:
- id: flake8

Expand All @@ -57,7 +57,7 @@ repos:
# compatible with 'black' with the lines set to ensure so in the repo's
# pyproject.toml. Other than that and the below, no extra config is required.
- repo: https://github.com/pycqa/isort
rev: 5.8.0
rev: 5.10.1
hooks:
- id: isort
name: isort (python)
Expand Down
65 changes: 11 additions & 54 deletions cf/data/collapse.py
Original file line number Diff line number Diff line change
Expand Up @@ -1137,11 +1137,7 @@ def cf_mean_chunk(x, weights=None, dtype="f8", computing_meta=False, **kwargs):


def cf_mean_combine(
pairs,
axis=None,
dtype="f8",
computing_meta=False,
**kwargs,
pairs, axis=None, dtype="f8", computing_meta=False, **kwargs
):
"""Combination calculations for the mean.

Expand Down Expand Up @@ -1255,12 +1251,7 @@ def cf_max_chunk(x, dtype=None, computing_meta=False, **kwargs):
}


def cf_max_combine(
pairs,
axis=None,
computing_meta=False,
**kwargs,
):
def cf_max_combine(pairs, axis=None, computing_meta=False, **kwargs):
"""Combination calculations for the maximum.

This function is passed to `dask.array.reduction` as its *combine*
Expand All @@ -1284,10 +1275,7 @@ def cf_max_combine(
if computing_meta:
return mx

return {
"max": mx,
"N": sum_sample_sizes(pairs, axis, **kwargs),
}
return {"max": mx, "N": sum_sample_sizes(pairs, axis, **kwargs)}


def cf_max_agg(
Expand Down Expand Up @@ -1414,12 +1402,7 @@ def cf_min_chunk(x, dtype=None, computing_meta=False, **kwargs):
}


def cf_min_combine(
pairs,
axis=None,
computing_meta=False,
**kwargs,
):
def cf_min_combine(pairs, axis=None, computing_meta=False, **kwargs):
"""Combination calculations for the minimum.

This function is passed to `dask.array.reduction` as its *combine*
Expand All @@ -1443,10 +1426,7 @@ def cf_min_combine(
if computing_meta:
return mn

return {
"min": mn,
"N": sum_sample_sizes(pairs, axis, **kwargs),
}
return {"min": mn, "N": sum_sample_sizes(pairs, axis, **kwargs)}


def cf_min_agg(
Expand Down Expand Up @@ -1528,11 +1508,7 @@ def cf_range_chunk(x, dtype=None, computing_meta=False, **kwargs):


def cf_range_combine(
pairs,
axis=None,
dtype=None,
computing_meta=False,
**kwargs,
pairs, axis=None, dtype=None, computing_meta=False, **kwargs
):
"""Combination calculations for the range.

Expand All @@ -1559,11 +1535,7 @@ def cf_range_combine(

mn = min_arrays(pairs, "min", axis, None, **kwargs)

return {
"max": mx,
"min": mn,
"N": sum_sample_sizes(pairs, axis, **kwargs),
}
return {"max": mx, "min": mn, "N": sum_sample_sizes(pairs, axis, **kwargs)}


def cf_range_agg(
Expand Down Expand Up @@ -1724,11 +1696,7 @@ def cf_sample_size_chunk(x, dtype="i8", computing_meta=False, **kwargs):


def cf_sample_size_combine(
pairs,
axis=None,
dtype="i8",
computing_meta=False,
**kwargs,
pairs, axis=None, dtype="i8", computing_meta=False, **kwargs
):
"""Combination calculations for the sample size.

Expand Down Expand Up @@ -1836,11 +1804,7 @@ def cf_sum_chunk(x, weights=None, dtype="f8", computing_meta=False, **kwargs):


def cf_sum_combine(
pairs,
axis=None,
dtype="f8",
computing_meta=False,
**kwargs,
pairs, axis=None, dtype="f8", computing_meta=False, **kwargs
):
"""Combination calculations for the sum.

Expand All @@ -1865,10 +1829,7 @@ def cf_sum_combine(
if computing_meta:
return x

return {
"sum": x,
"N": sum_sample_sizes(pairs, axis, **kwargs),
}
return {"sum": x, "N": sum_sample_sizes(pairs, axis, **kwargs)}


def cf_sum_agg(
Expand Down Expand Up @@ -2099,11 +2060,7 @@ def cf_var_chunk(


def cf_var_combine(
pairs,
axis=None,
dtype="f8",
computing_meta=False,
**kwargs,
pairs, axis=None, dtype="f8", computing_meta=False, **kwargs
):
"""Combination calculations for the variance.

Expand Down
16 changes: 2 additions & 14 deletions cf/data/dask_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,26 +93,14 @@ def allclose(a_blocks, b_blocks, rtol=rtol, atol=atol):

for a, b in zip(flatten(a_blocks), flatten(b_blocks)):
result &= np.ma.allclose(
a,
b,
masked_equal=masked_equal,
rtol=rtol,
atol=atol,
a, b, masked_equal=masked_equal, rtol=rtol, atol=atol
)

return result

axes = tuple(range(x.ndim))
return da.blockwise(
allclose,
"",
x,
axes,
y,
axes,
dtype=bool,
rtol=rtol,
atol=atol,
allclose, "", x, axes, y, axes, dtype=bool, rtol=rtol, atol=atol
)


Expand Down
Loading