From 7b39e6215fdb831ac6b299c1019f9f7bfe22ab9a Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Thu, 8 Dec 2022 23:47:38 -0600 Subject: [PATCH 01/26] Introduce maybe_posaxis as a replacement for axis_wrap_if_negative and implement ak.is_none. --- src/awkward/_do.py | 16 +++++ src/awkward/operations/ak_is_none.py | 76 ++++++++------------- tests/test_1914-improved-axis-to-posaxis.py | 67 ++++++++++++++++++ 3 files changed, 113 insertions(+), 46 deletions(-) create mode 100644 tests/test_1914-improved-axis-to-posaxis.py diff --git a/src/awkward/_do.py b/src/awkward/_do.py index fd4a749544..83b31922bb 100644 --- a/src/awkward/_do.py +++ b/src/awkward/_do.py @@ -122,6 +122,22 @@ def getkey(layout, form, attribute): return form, len(content), container +# FIXME: eventually, this should be in ak._util, but it's here to juxtapose axis_wrap_if_negative +# This function must depend on 'depth'. I don't know how axis_wrap_if_negative was believed to work. +def maybe_posaxis( + layout: Content, axis: AxisMaybeNone, depth: Integral +) -> AxisMaybeNone: + if axis >= 0: + return axis + else: + is_branching, additional_depth = layout.branch_depth + if not is_branching: + return axis + depth + additional_depth - 1 + else: + return None + + +# FIXME: eventually phase out the use of axis_wrap_if_negative in favor of maybe_posaxis def axis_wrap_if_negative( layout: Content | Record, axis: AxisMaybeNone ) -> AxisMaybeNone: diff --git a/src/awkward/operations/ak_is_none.py b/src/awkward/operations/ak_is_none.py index cfac4095d9..5f28459b34 100644 --- a/src/awkward/operations/ak_is_none.py +++ b/src/awkward/operations/ak_is_none.py @@ -29,59 +29,43 @@ def is_none(array, axis=0, *, highlevel=True, behavior=None): def _impl(array, axis, highlevel, behavior): + layout = ak.operations.to_layout(array) + behavior = ak._util.behavior_of(array, behavior=behavior) - # Determine the (potentially nested) bytemask - def getfunction_inner(layout, depth, **kwargs): - if not isinstance(layout, ak.contents.Content): - return + if not ak._util.is_integer(axis): + raise ak._errors.wrap_error( + TypeError(f"'axis' must be an integer, not {axis!r}") + ) - backend = layout.backend + def action(layout, depth, **kwargs): + posaxis = ak._do.maybe_posaxis(layout, axis, depth) - if layout.is_option: - layout = layout.to_IndexedOptionArray64() + if posaxis is not None and posaxis + 1 == depth: + if layout.is_union: + contents = [ + ak._do.recursively_apply(x, action, behavior, numpy_to_regular=True) + for x in layout.contents + ] + return ak.contents.UnionArray.simplified( + layout.tags, + layout.index, + contents, + ) - # Convert the option type into a union, using the mask - # as a tag. - tag = backend.index_nplike.asarray(layout.mask_as_bool(valid_when=False)) - index = backend.index_nplike.where( - tag, 0, backend.nplike.asarray(layout.index) - ) + elif layout.is_option: + return ak.contents.NumpyArray(layout.mask_as_bool(valid_when=False)) - return ak.contents.UnionArray.simplified( - ak.index.Index8(tag), - ak.index.Index64(index), - [ - ak._do.recursively_apply( - layout.content, getfunction_inner, behavior - ), - ak.contents.NumpyArray( - backend.nplike.array([True], dtype=np.bool_) - ), - ], - ) + else: + nplike = layout._backend.nplike + return ak.contents.NumpyArray( + nplike.zeros(layout.length, dtype=np.bool_) + ) - elif layout.is_numpy or layout.is_unknown or layout.is_list or layout.is_record: - return ak.contents.NumpyArray( - backend.nplike.zeros(len(layout), dtype=np.bool_) + elif layout.is_leaf: + raise ak._errors.wrap_error( + np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") ) - # Locate the axis - def getfunction_outer(layout, depth, depth_context, **kwargs): - depth_context["posaxis"] = ak._do.axis_wrap_if_negative( - layout, depth_context["posaxis"] - ) - if depth_context["posaxis"] == depth - 1: - return ak._do.recursively_apply(layout, getfunction_inner, behavior) + out = ak._do.recursively_apply(layout, action, behavior, numpy_to_regular=True) - layout = ak.operations.to_layout(array) - max_axis = layout.branch_depth[1] - 1 - if axis > max_axis: - raise ak._errors.wrap_error( - np.AxisError(f"axis={axis} exceeds the depth of this array ({max_axis})") - ) - behavior = ak._util.behavior_of(array, behavior=behavior) - depth_context = {"posaxis": axis} - out = ak._do.recursively_apply( - layout, getfunction_outer, behavior, depth_context=depth_context - ) return ak._util.wrap(out, behavior, highlevel) diff --git a/tests/test_1914-improved-axis-to-posaxis.py b/tests/test_1914-improved-axis-to-posaxis.py new file mode 100644 index 0000000000..a1160d320b --- /dev/null +++ b/tests/test_1914-improved-axis-to-posaxis.py @@ -0,0 +1,67 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE + +import numpy as np +import pytest + +import awkward as ak + + +def test_is_none(): + array = ak.Array( + [ + None, + [None], + [{"x": None, "y": None}], + [{"x": [None], "y": [None]}], + [{"x": [1], "y": [[None]]}], + [{"x": [2], "y": [[1, 2, 3]]}], + ] + ) + + assert ak.is_none(array, axis=0).tolist() == [ + True, + False, + False, + False, + False, + False, + ] + assert ak.is_none(array, axis=1).tolist() == [ + None, + [True], + [False], + [False], + [False], + [False], + ] + assert ak.is_none(array, axis=2).tolist() == [ + None, + [None], + [{"x": None, "y": None}], + [{"x": [True], "y": [True]}], + [{"x": [False], "y": [False]}], + [{"x": [False], "y": [False]}], + ] + + with pytest.raises(np.AxisError): + ak.is_none(array, axis=3) + + assert ak.is_none(array, axis=-1).tolist() == [ + None, + [None], + [{"x": None, "y": None}], + [{"x": [True], "y": [None]}], + [{"x": [False], "y": [[True]]}], + [{"x": [False], "y": [[False, False, False]]}], + ] + assert ak.is_none(array, axis=-2).tolist() == [ + None, + [None], + [{"x": True, "y": None}], + [{"x": False, "y": [True]}], + [{"x": False, "y": [False]}], + [{"x": False, "y": [False]}], + ] + + with pytest.raises(np.AxisError): + ak.is_none(array, axis=-3) From deb912e38244197f921c5674465e0ba79af95d19 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 00:37:11 -0600 Subject: [PATCH 02/26] Use maybe_posaxis in ak.singletons. --- src/awkward/operations/ak_singletons.py | 15 ++--- tests/test_1914-improved-axis-to-posaxis.py | 64 +++++++++++++++++++++ 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/src/awkward/operations/ak_singletons.py b/src/awkward/operations/ak_singletons.py index 3331a28837..9ea1f78bd0 100644 --- a/src/awkward/operations/ak_singletons.py +++ b/src/awkward/operations/ak_singletons.py @@ -46,16 +46,16 @@ def singletons(array, axis=0, *, highlevel=True, behavior=None): def _impl(array, axis, highlevel, behavior): layout = ak.operations.to_layout(array) behavior = ak._util.behavior_of(array, behavior=behavior) - posaxis = ak._do.axis_wrap_if_negative(layout, axis) if not ak._util.is_integer(axis): raise ak._errors.wrap_error( TypeError(f"'axis' must be an integer, not {axis!r}") ) - def action(layout, depth, depth_context, **kwargs): - posaxis = ak._do.axis_wrap_if_negative(layout, depth_context["posaxis"]) - if posaxis >= 0 and posaxis + 1 == depth: + def action(layout, depth, **kwargs): + posaxis = ak._do.maybe_posaxis(layout, axis, depth) + + if posaxis is not None and posaxis + 1 == depth: if layout.is_option: nplike = layout._backend.index_nplike @@ -76,11 +76,6 @@ def action(layout, depth, depth_context, **kwargs): np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") ) - depth_context["posaxis"] = posaxis - - depth_context = {"posaxis": posaxis} - out = ak._do.recursively_apply( - layout, action, behavior, depth_context, numpy_to_regular=True - ) + out = ak._do.recursively_apply(layout, action, behavior, numpy_to_regular=True) return ak._util.wrap(out, behavior, highlevel) diff --git a/tests/test_1914-improved-axis-to-posaxis.py b/tests/test_1914-improved-axis-to-posaxis.py index a1160d320b..38e73046e0 100644 --- a/tests/test_1914-improved-axis-to-posaxis.py +++ b/tests/test_1914-improved-axis-to-posaxis.py @@ -65,3 +65,67 @@ def test_is_none(): with pytest.raises(np.AxisError): ak.is_none(array, axis=-3) + + +def test_singletons(): + array = ak.Array( + [ + None, + [None], + [{"x": None, "y": None}], + [{"x": [None], "y": [None]}], + [{"x": [1], "y": [[None]]}], + [{"x": [2], "y": [[1, 2, 3]]}], + ] + ) + + assert ak.singletons(array, axis=0).tolist() == [ + [], + [[None]], + [[{"x": None, "y": None}]], + [[{"x": [None], "y": [None]}]], + [[{"x": [1], "y": [[None]]}]], + [[{"x": [2], "y": [[1, 2, 3]]}]], + ] + + assert ak.singletons(array, axis=1).tolist() == [ + None, + [[]], + [[{"x": None, "y": None}]], + [[{"x": [None], "y": [None]}]], + [[{"x": [1], "y": [[None]]}]], + [[{"x": [2], "y": [[1, 2, 3]]}]], + ] + + assert ak.singletons(array, axis=2).tolist() == [ + None, + [None], + [{"x": None, "y": None}], + [{"x": [[]], "y": [[]]}], + [{"x": [[1]], "y": [[[None]]]}], + [{"x": [[2]], "y": [[[1, 2, 3]]]}], + ] + + with pytest.raises(np.AxisError): + ak.singletons(array, axis=3) + + assert ak.singletons(array, axis=-1).tolist() == [ + None, + [None], + [{"x": None, "y": None}], + [{"x": [[]], "y": [None]}], + [{"x": [[1]], "y": [[[]]]}], + [{"x": [[2]], "y": [[[1], [2], [3]]]}], + ] + + assert ak.singletons(array, axis=-2).tolist() == [ + None, + [None], + [{"x": [], "y": None}], + [{"x": [[None]], "y": [[]]}], + [{"x": [[1]], "y": [[[None]]]}], + [{"x": [[2]], "y": [[[1, 2, 3]]]}], + ] + + with pytest.raises(np.AxisError): + ak.singletons(array, axis=-3) From 8a29851d6d231533336071b4637719d1c0daceb8 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 00:38:44 -0600 Subject: [PATCH 03/26] Also handle is_union case in ak.singletons. --- src/awkward/operations/ak_singletons.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/awkward/operations/ak_singletons.py b/src/awkward/operations/ak_singletons.py index 9ea1f78bd0..69ea4aab41 100644 --- a/src/awkward/operations/ak_singletons.py +++ b/src/awkward/operations/ak_singletons.py @@ -56,7 +56,18 @@ def action(layout, depth, **kwargs): posaxis = ak._do.maybe_posaxis(layout, axis, depth) if posaxis is not None and posaxis + 1 == depth: - if layout.is_option: + if layout.is_union: + contents = [ + ak._do.recursively_apply(x, action, behavior, numpy_to_regular=True) + for x in layout.contents + ] + return ak.contents.UnionArray.simplified( + layout.tags, + layout.index, + contents, + ) + + elif layout.is_option: nplike = layout._backend.index_nplike offsets = nplike.empty(layout.length + 1, dtype=np.int64) From d3c54b72e7e711c471da360f2d54f6e3527c3d0e Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 00:42:08 -0600 Subject: [PATCH 04/26] Use maybe_posaxis in ak.firsts. --- src/awkward/operations/ak_firsts.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/awkward/operations/ak_firsts.py b/src/awkward/operations/ak_firsts.py index bffb83183e..f7c9d7a6b5 100644 --- a/src/awkward/operations/ak_firsts.py +++ b/src/awkward/operations/ak_firsts.py @@ -46,14 +46,13 @@ def firsts(array, axis=1, *, highlevel=True, behavior=None): def _impl(array, axis, highlevel, behavior): layout = ak.operations.to_layout(array) behavior = ak._util.behavior_of(array, behavior=behavior) - posaxis = ak._do.axis_wrap_if_negative(layout, axis) if not ak._util.is_integer(axis): raise ak._errors.wrap_error( TypeError(f"'axis' must be an integer, not {axis!r}") ) - if posaxis == 0: + if ak._do.maybe_posaxis(layout, axis, 1) == 0: # specialized logic; it's tested in test_0582-propagate-context-in-broadcast_and_apply.py # Build an integer-typed slice array, so that we can # ensure we have advanced indexing for both length==0 @@ -67,7 +66,8 @@ def _impl(array, axis, highlevel, behavior): else: def action(layout, depth, depth_context, **kwargs): - posaxis = ak._do.axis_wrap_if_negative(layout, depth_context["posaxis"]) + posaxis = ak._do.maybe_posaxis(layout, axis, depth) + if posaxis == depth and layout.is_list: nplike = layout._backend.index_nplike @@ -91,11 +91,6 @@ def action(layout, depth, depth_context, **kwargs): ) ) - depth_context["posaxis"] = posaxis - - depth_context = {"posaxis": posaxis} - out = ak._do.recursively_apply( - layout, action, behavior, depth_context, numpy_to_regular=True - ) + out = ak._do.recursively_apply(layout, action, behavior, numpy_to_regular=True) return ak._util.wrap(out, behavior, highlevel) From 5ddb4308247b35f3862c14c461256495604c8525 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 00:47:30 -0600 Subject: [PATCH 05/26] Use maybe_posaxis in ak.unflatten. --- src/awkward/operations/ak_unflatten.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/awkward/operations/ak_unflatten.py b/src/awkward/operations/ak_unflatten.py index 7ed49ba035..b9c0894c4f 100644 --- a/src/awkward/operations/ak_unflatten.py +++ b/src/awkward/operations/ak_unflatten.py @@ -156,18 +156,18 @@ def doit(layout): return out - if axis == 0 or ak._do.axis_wrap_if_negative(layout, axis) == 0: + if axis == 0 or ak._do.maybe_posaxis(layout, axis, 1) == 0: out = doit(layout) else: - def transform(layout, depth, posaxis): + def transform(layout, depth, axis): # Pack the current layout. This ensures that the `counts` array, # which is computed with these layouts applied, aligns with the # internal layout to be unflattened (#910) layout = layout.to_packed() - posaxis = ak._do.axis_wrap_if_negative(layout, posaxis) + posaxis = ak._do.maybe_posaxis(layout, axis, depth) if posaxis == depth and layout.is_list: # We are one *above* the level where we want to apply this. listoffsetarray = layout.to_ListOffsetArray64(True) @@ -205,7 +205,7 @@ def transform(layout, depth, posaxis): else: return layout - out = transform(layout, depth=1, posaxis=axis) + out = transform(layout, depth=1, axis=axis) if current_offsets is not None and not ( len(current_offsets[0]) == 1 and current_offsets[0][0] == 0 From 89c12e2ece24dce96e784fbd8ed10fbb011dc005 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 00:50:16 -0600 Subject: [PATCH 06/26] Use maybe_posaxis in ak.from_regular. --- src/awkward/operations/ak_from_regular.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/awkward/operations/ak_from_regular.py b/src/awkward/operations/ak_from_regular.py index b907462f51..124a5f11dd 100644 --- a/src/awkward/operations/ak_from_regular.py +++ b/src/awkward/operations/ak_from_regular.py @@ -43,7 +43,6 @@ def from_regular(array, axis=1, *, highlevel=True, behavior=None): def _impl(array, axis, highlevel, behavior): layout = ak.operations.to_layout(array) behavior = ak._util.behavior_of(array, behavior=behavior) - posaxis = ak._do.axis_wrap_if_negative(layout, axis) if axis is None: @@ -53,13 +52,13 @@ def action(layout, continuation, **kwargs): out = ak._do.recursively_apply(layout, action, behavior, numpy_to_regular=True) - elif posaxis == 0: + elif ak._do.maybe_posaxis(layout, axis, 1) == 0: out = layout # the top-level is already regular (ArrayType) else: - def action(layout, depth, depth_context, **kwargs): - posaxis = ak._do.axis_wrap_if_negative(layout, depth_context["posaxis"]) + def action(layout, depth, **kwargs): + posaxis = ak._do.maybe_posaxis(layout, axis, depth) if posaxis == depth and layout.is_regular: return layout.to_ListOffsetArray64(False) elif posaxis == depth and layout.is_list: @@ -71,11 +70,6 @@ def action(layout, depth, depth_context, **kwargs): ) ) - depth_context["posaxis"] = posaxis - - depth_context = {"posaxis": posaxis} - out = ak._do.recursively_apply( - layout, action, behavior, depth_context, numpy_to_regular=True - ) + out = ak._do.recursively_apply(layout, action, behavior, numpy_to_regular=True) return ak._util.wrap(out, behavior, highlevel) From ccdc8ea7ada1dcec462337351c024146a79a162b Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 00:54:54 -0600 Subject: [PATCH 07/26] Use maybe_posaxis in ak.to_regular. --- src/awkward/_do.py | 10 +++++++++- src/awkward/operations/ak_to_regular.py | 12 ++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/awkward/_do.py b/src/awkward/_do.py index 83b31922bb..5d38f0872c 100644 --- a/src/awkward/_do.py +++ b/src/awkward/_do.py @@ -125,10 +125,18 @@ def getkey(layout, form, attribute): # FIXME: eventually, this should be in ak._util, but it's here to juxtapose axis_wrap_if_negative # This function must depend on 'depth'. I don't know how axis_wrap_if_negative was believed to work. def maybe_posaxis( - layout: Content, axis: AxisMaybeNone, depth: Integral + layout: Content | Record, axis: AxisMaybeNone, depth: Integral ) -> AxisMaybeNone: + if isinstance(layout, Record): + if axis == 0: + raise ak._errors.wrap_error( + np.AxisError("Record type at axis=0 is a scalar, not an array") + ) + return maybe_posaxis(layout._array, axis, depth) + if axis >= 0: return axis + else: is_branching, additional_depth = layout.branch_depth if not is_branching: diff --git a/src/awkward/operations/ak_to_regular.py b/src/awkward/operations/ak_to_regular.py index 6a4c9d6f98..180b2775ad 100644 --- a/src/awkward/operations/ak_to_regular.py +++ b/src/awkward/operations/ak_to_regular.py @@ -56,7 +56,6 @@ def to_regular(array, axis=1, *, highlevel=True, behavior=None): def _impl(array, axis, highlevel, behavior): layout = ak.operations.to_layout(array) behavior = ak._util.behavior_of(array, behavior=behavior) - posaxis = ak._do.axis_wrap_if_negative(layout, axis) if axis is None: @@ -66,13 +65,13 @@ def action(layout, continuation, **kwargs): out = ak._do.recursively_apply(layout, action, behavior) - elif posaxis == 0: + elif ak._do.maybe_posaxis(layout, axis, 1) == 0: out = layout # the top-level can only be regular (ArrayType) else: - def action(layout, depth, depth_context, **kwargs): - posaxis = ak._do.axis_wrap_if_negative(layout, depth_context["posaxis"]) + def action(layout, depth, **kwargs): + posaxis = ak._do.maybe_posaxis(layout, axis, depth) if posaxis == depth and layout.is_list: return layout.to_RegularArray() elif posaxis == 0: @@ -82,9 +81,6 @@ def action(layout, depth, depth_context, **kwargs): ) ) - depth_context["posaxis"] = posaxis - - depth_context = {"posaxis": posaxis} - out = ak._do.recursively_apply(layout, action, behavior, depth_context) + out = ak._do.recursively_apply(layout, action, behavior) return ak._util.wrap(out, behavior, highlevel) From a8c04cd659ce5b1c9896f427aeff10851692675a Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 01:02:38 -0600 Subject: [PATCH 08/26] ak.to_regular had been silently exceeding the array. --- src/awkward/operations/ak_to_regular.py | 3 ++- tests/test_1565-axis_wrap_if_negative_record.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/awkward/operations/ak_to_regular.py b/src/awkward/operations/ak_to_regular.py index 180b2775ad..7ac65a8880 100644 --- a/src/awkward/operations/ak_to_regular.py +++ b/src/awkward/operations/ak_to_regular.py @@ -74,7 +74,8 @@ def action(layout, depth, **kwargs): posaxis = ak._do.maybe_posaxis(layout, axis, depth) if posaxis == depth and layout.is_list: return layout.to_RegularArray() - elif posaxis == 0: + + elif layout.is_leaf: raise ak._errors.wrap_error( np.AxisError( f"axis={axis} exceeds the depth of this array ({depth})" diff --git a/tests/test_1565-axis_wrap_if_negative_record.py b/tests/test_1565-axis_wrap_if_negative_record.py index 2f1ac4a4a2..1048ed70ce 100644 --- a/tests/test_1565-axis_wrap_if_negative_record.py +++ b/tests/test_1565-axis_wrap_if_negative_record.py @@ -26,7 +26,6 @@ def test_axis_wrap_if_negative_record_v2(): with pytest.raises(np.AxisError): r = ak.operations.to_regular(r, 0) - r = ak.operations.to_regular(r, 2) list_cell_chain_field = [ [["TRA", "TRAV1", 15], ["TRB", "TRBV1", 12]], From 6b1271edeb6235c8ca57d8a98df268b11a04f479 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 01:06:25 -0600 Subject: [PATCH 09/26] Protect ak.fill_none as well. --- src/awkward/operations/ak_fill_none.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/awkward/operations/ak_fill_none.py b/src/awkward/operations/ak_fill_none.py index 1c17560e33..f8b617e0bc 100644 --- a/src/awkward/operations/ak_fill_none.py +++ b/src/awkward/operations/ak_fill_none.py @@ -119,17 +119,19 @@ def action(layout, continuation, **kwargs): else: - def action(layout, depth, depth_context, **kwargs): - posaxis = ak._do.axis_wrap_if_negative(layout, depth_context["posaxis"]) - depth_context["posaxis"] = posaxis - if posaxis + 1 < depth: + def action(layout, depth, **kwargs): + posaxis = ak._do.maybe_posaxis(layout, axis, depth) + if posaxis is not None and posaxis + 1 < depth: return layout - elif posaxis + 1 == depth: + elif posaxis is not None and posaxis + 1 == depth: return maybe_fillna(layout) - - depth_context = {"posaxis": axis} - out = ak._do.recursively_apply( - arraylayout, action, behavior, depth_context=depth_context - ) + elif layout.is_leaf: + raise ak._errors.wrap_error( + np.AxisError( + f"axis={axis} exceeds the depth of this array ({depth})" + ) + ) + + out = ak._do.recursively_apply(arraylayout, action, behavior) return ak._util.wrap(out, behavior, highlevel) From aa42b9b976d4c52368c5269d3dc0441e391395aa Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 01:10:38 -0600 Subject: [PATCH 10/26] Use maybe_posaxis in ak.cartesian. --- src/awkward/operations/ak_cartesian.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/awkward/operations/ak_cartesian.py b/src/awkward/operations/ak_cartesian.py index 298cf9b196..d95b86e706 100644 --- a/src/awkward/operations/ak_cartesian.py +++ b/src/awkward/operations/ak_cartesian.py @@ -235,11 +235,11 @@ def _impl(arrays, axis, nested, parameters, with_name, highlevel, behavior): else: new_arrays_values = new_arrays - posaxis = ak._do.axis_wrap_if_negative(new_arrays_values[0], axis) - if posaxis < 0: + posaxis = ak._do.maybe_posaxis(new_arrays_values[0], axis, 1) + if posaxis is None or posaxis < 0: raise ak._errors.wrap_error(ValueError("negative axis depth is ambiguous")) for x in new_arrays_values[1:]: - if ak._do.axis_wrap_if_negative(x, axis) != posaxis: + if ak._do.maybe_posaxis(x, axis, 1) != posaxis: raise ak._errors.wrap_error( ValueError( "arrays to cartesian-product do not have the same depth for negative axis" From 353c7802cb48e47c3d439710a1ec2f1cdc5af5f8 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 01:12:00 -0600 Subject: [PATCH 11/26] Use maybe_posaxis in ak.flatten. --- src/awkward/operations/ak_flatten.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/awkward/operations/ak_flatten.py b/src/awkward/operations/ak_flatten.py index 4fca695a82..85a17440fe 100644 --- a/src/awkward/operations/ak_flatten.py +++ b/src/awkward/operations/ak_flatten.py @@ -174,7 +174,7 @@ def _impl(array, axis, highlevel, behavior): return ak._util.wrap(result, behavior, highlevel) - elif axis == 0 or ak._do.axis_wrap_if_negative(layout, axis) == 0: + elif axis == 0 or ak._do.maybe_posaxis(layout, axis, 1) == 0: def apply(layout): backend = layout.backend From 0ff4ec0b9cead993e4f2fb43d84d955d52742534 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 01:12:46 -0600 Subject: [PATCH 12/26] Use maybe_posaxis in ak.ptp. --- src/awkward/operations/ak_ptp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/awkward/operations/ak_ptp.py b/src/awkward/operations/ak_ptp.py index 7643c61a9b..77531d0ea4 100644 --- a/src/awkward/operations/ak_ptp.py +++ b/src/awkward/operations/ak_ptp.py @@ -131,7 +131,7 @@ def _impl(array, axis, keepdims, mask_identity, flatten_records): out = ak.highlevel.Array(ak.operations.fill_none(out, 0, axis=-1)) if not keepdims: - posaxis = ak._do.axis_wrap_if_negative(out.layout, axis) + posaxis = ak._do.maybe_posaxis(out.layout, axis, 1) out = out[(slice(None, None),) * posaxis + (0,)] return out From cb5e490cc486aafdb86dbbb4ef4529ad2d523168 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 01:16:53 -0600 Subject: [PATCH 13/26] Use maybe_posaxis in ak.concatenate. --- src/awkward/operations/ak_concatenate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/awkward/operations/ak_concatenate.py b/src/awkward/operations/ak_concatenate.py index de7164bd2b..465f9ba52e 100644 --- a/src/awkward/operations/ak_concatenate.py +++ b/src/awkward/operations/ak_concatenate.py @@ -56,7 +56,7 @@ def _impl(arrays, axis, mergebool, highlevel, behavior): content = ak.operations.to_layout(arrays, allow_record=False, allow_other=False) # Only handle concatenation along `axis=0` # Let ambiguous depth arrays fall through - if ak._do.axis_wrap_if_negative(content, axis) == 0: + if ak._do.maybe_posaxis(content, axis, 1) == 0: return ak.operations.ak_flatten._impl(content, 1, highlevel, behavior) content_or_others = [ @@ -72,13 +72,13 @@ def _impl(arrays, axis, mergebool, highlevel, behavior): ValueError("need at least one array to concatenate") ) - posaxis = ak._do.axis_wrap_if_negative(contents[0], axis) + posaxis = ak._do.maybe_posaxis(contents[0], axis, 1) maxdepth = max( x.minmax_depth[1] for x in content_or_others if isinstance(x, ak.contents.Content) ) - if not 0 <= posaxis < maxdepth: + if posaxis is None or not 0 <= posaxis < maxdepth: raise ak._errors.wrap_error( ValueError( "axis={} is beyond the depth of this array or the depth of this array " @@ -87,7 +87,7 @@ def _impl(arrays, axis, mergebool, highlevel, behavior): ) for x in content_or_others: if isinstance(x, ak.contents.Content): - if ak._do.axis_wrap_if_negative(x, axis) != posaxis: + if ak._do.maybe_posaxis(x, axis, 1) != posaxis: raise ak._errors.wrap_error( ValueError( "arrays to concatenate do not have the same depth for negative " From 9fd558cda083e224c2fa5f8e76552cdf401932ca Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 01:36:05 -0600 Subject: [PATCH 14/26] The 'num' function actually takes 'depth minus 1'. --- src/awkward/_do.py | 2 +- src/awkward/contents/bitmaskedarray.py | 4 ++-- src/awkward/contents/bytemaskedarray.py | 6 +++--- src/awkward/contents/emptyarray.py | 4 ++-- src/awkward/contents/indexedarray.py | 6 +++--- src/awkward/contents/indexedoptionarray.py | 6 +++--- src/awkward/contents/listarray.py | 8 ++++---- src/awkward/contents/listoffsetarray.py | 8 ++++---- src/awkward/contents/numpyarray.py | 12 ++++++------ src/awkward/contents/recordarray.py | 6 +++--- src/awkward/contents/regulararray.py | 8 ++++---- src/awkward/contents/unionarray.py | 6 +++--- src/awkward/contents/unmaskedarray.py | 6 +++--- tests/test_0163-negative-axis-wrap.py | 12 ++++++------ tests/test_1137-num.py | 16 ++++++++-------- 15 files changed, 55 insertions(+), 55 deletions(-) diff --git a/src/awkward/_do.py b/src/awkward/_do.py index 5d38f0872c..4035faecfc 100644 --- a/src/awkward/_do.py +++ b/src/awkward/_do.py @@ -319,7 +319,7 @@ def fill_none(layout: Content, value: Content) -> Content: def num(layout, axis): - return layout._num(axis) + return layout._num(axis, 0) def mergeable(one: Content, two: Content, mergebool: bool = True) -> bool: diff --git a/src/awkward/contents/bitmaskedarray.py b/src/awkward/contents/bitmaskedarray.py index 4233f858cc..0200df0aec 100644 --- a/src/awkward/contents/bitmaskedarray.py +++ b/src/awkward/contents/bitmaskedarray.py @@ -434,8 +434,8 @@ def _getitem_next(self, head, tail, advanced): def project(self, mask=None): return self.to_ByteMaskedArray().project(mask) - def _num(self, axis, depth=0): - return self.to_ByteMaskedArray()._num(axis, depth) + def _num(self, axis, depth_m1): + return self.to_ByteMaskedArray()._num(axis, depth_m1) def _offsets_and_flattened(self, axis, depth): return self.to_ByteMaskedArray._offsets_and_flattened(axis, depth) diff --git a/src/awkward/contents/bytemaskedarray.py b/src/awkward/contents/bytemaskedarray.py index de564dbb78..81d17a0475 100644 --- a/src/awkward/contents/bytemaskedarray.py +++ b/src/awkward/contents/bytemaskedarray.py @@ -556,9 +556,9 @@ def project(self, mask=None): return self._content._carry(nextcarry, False) - def _num(self, axis, depth=0): + def _num(self, axis, depth_m1): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis == depth_m1: out = self.length if ak._util.is_integer(out): return np.int64(out) @@ -568,7 +568,7 @@ def _num(self, axis, depth=0): _, nextcarry, outindex = self._nextcarry_outindex(self._backend) next = self._content._carry(nextcarry, False) - out = next._num(posaxis, depth) + out = next._num(posaxis, depth_m1) return ak.contents.IndexedOptionArray.simplified( outindex, out, parameters=self._parameters diff --git a/src/awkward/contents/emptyarray.py b/src/awkward/contents/emptyarray.py index 0ce2efaa61..cd634fe86d 100644 --- a/src/awkward/contents/emptyarray.py +++ b/src/awkward/contents/emptyarray.py @@ -158,10 +158,10 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth=0): + def _num(self, axis, depth_m1): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis == depth_m1: out = self.length if ak._util.is_integer(out): return np.int64(out) diff --git a/src/awkward/contents/indexedarray.py b/src/awkward/contents/indexedarray.py index 456b965f7f..a2eb1d0273 100644 --- a/src/awkward/contents/indexedarray.py +++ b/src/awkward/contents/indexedarray.py @@ -388,16 +388,16 @@ def project(self, mask=None): ) ) - def _num(self, axis, depth=0): + def _num(self, axis, depth_m1): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis == depth_m1: out = self.length if ak._util.is_integer(out): return np.int64(out) else: return out else: - return self.project()._num(posaxis, depth) + return self.project()._num(posaxis, depth_m1) def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) diff --git a/src/awkward/contents/indexedoptionarray.py b/src/awkward/contents/indexedoptionarray.py index 6a4d20f5d3..84f4321726 100644 --- a/src/awkward/contents/indexedoptionarray.py +++ b/src/awkward/contents/indexedoptionarray.py @@ -475,9 +475,9 @@ def project(self, mask=None): return self._content._carry(nextcarry, False) - def _num(self, axis, depth=0): + def _num(self, axis, depth_m1): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis == depth_m1: out = self.length if ak._util.is_integer(out): return np.int64(out) @@ -485,7 +485,7 @@ def _num(self, axis, depth=0): return out _, nextcarry, outindex = self._nextcarry_outindex(self._backend) next = self._content._carry(nextcarry, False) - out = next._num(posaxis, depth) + out = next._num(posaxis, depth_m1) return ak.contents.IndexedOptionArray.simplified( outindex, out, parameters=self._parameters ) diff --git a/src/awkward/contents/listarray.py b/src/awkward/contents/listarray.py index 2253c3b925..d5ab7c9b83 100644 --- a/src/awkward/contents/listarray.py +++ b/src/awkward/contents/listarray.py @@ -894,15 +894,15 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth=0): + def _num(self, axis, depth_m1): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis == depth_m1: out = self.length if ak._util.is_integer(out): return np.int64(out) else: return out - elif posaxis == depth + 1: + elif posaxis == depth_m1 + 1: tonum = ak.index.Index64.empty(self.length, self._backend.index_nplike) assert ( tonum.nplike is self._backend.index_nplike @@ -924,7 +924,7 @@ def _num(self, axis, depth=0): ) return ak.contents.NumpyArray(tonum, parameters=None, backend=self._backend) else: - return self.to_ListOffsetArray64(True)._num(posaxis, depth) + return self.to_ListOffsetArray64(True)._num(posaxis, depth_m1) def _offsets_and_flattened(self, axis, depth): return self.to_ListOffsetArray64(True)._offsets_and_flattened(axis, depth) diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py index 04e9698479..120f805d74 100644 --- a/src/awkward/contents/listoffsetarray.py +++ b/src/awkward/contents/listoffsetarray.py @@ -601,15 +601,15 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth=0): + def _num(self, axis, depth_m1): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis == depth_m1: out = self.length if ak._util.is_integer(out): return np.int64(out) else: return out - elif posaxis == depth + 1: + elif posaxis == depth_m1 + 1: tonum = ak.index.Index64.empty(self.length, self._backend.index_nplike) assert ( tonum.nplike is self._backend.index_nplike @@ -631,7 +631,7 @@ def _num(self, axis, depth=0): ) return ak.contents.NumpyArray(tonum, parameters=None, backend=self._backend) else: - next = self._content._num(posaxis, depth + 1) + next = self._content._num(posaxis, depth_m1 + 1) offsets = self._compact_offsets64(True) return ak.contents.ListOffsetArray( offsets, next, parameters=self._parameters diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index 566d8d6c10..4d69c1ca68 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -315,9 +315,9 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth=0): + def _num(self, axis, depth_m1): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis == depth_m1: out = self.length if ak._util.is_integer(out): return np.int64(out) @@ -327,15 +327,15 @@ def _num(self, axis, depth=0): reps = 1 size = self.length i = 0 - while i < self._data.ndim - 1 and depth < posaxis: + while i < self._data.ndim - 1 and depth_m1 < posaxis: shape.append(self.shape[i]) reps *= self.shape[i] size = self.shape[i + 1] i += 1 - depth += 1 - if posaxis > depth: + depth_m1 += 1 + if posaxis > depth_m1: raise ak._errors.wrap_error( - np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") + np.AxisError(f"axis={axis} exceeds the depth of this array ({depth_m1 + 1})") ) tonum = ak.index.Index64.empty(reps, self._backend.index_nplike) diff --git a/src/awkward/contents/recordarray.py b/src/awkward/contents/recordarray.py index 71f14570c2..4878cf691b 100644 --- a/src/awkward/contents/recordarray.py +++ b/src/awkward/contents/recordarray.py @@ -443,9 +443,9 @@ def _getitem_next(self, head, tail, advanced): ) return next._getitem_next(nexthead, nexttail, advanced) - def _num(self, axis, depth=0): + def _num(self, axis, depth_m1): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis == depth_m1: npsingle = self._backend.index_nplike.full((1,), self.length, np.int64) single = ak.index.Index64(npsingle, nplike=self._backend.index_nplike) singleton = ak.contents.NumpyArray( @@ -463,7 +463,7 @@ def _num(self, axis, depth=0): else: contents = [] for content in self._contents: - contents.append(content._num(posaxis, depth)) + contents.append(content._num(posaxis, depth_m1)) return ak.contents.RecordArray( contents, self._fields, diff --git a/src/awkward/contents/regulararray.py b/src/awkward/contents/regulararray.py index c9c72e11fb..4bfd30f2e5 100644 --- a/src/awkward/contents/regulararray.py +++ b/src/awkward/contents/regulararray.py @@ -612,15 +612,15 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth=0): + def _num(self, axis, depth_m1): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis == depth_m1: out = self._length if ak._util.is_integer(out): return np.int64(out) else: return out - elif posaxis == depth + 1: + elif posaxis == depth_m1 + 1: tonum = ak.index.Index64.empty(self._length, self._backend.index_nplike) assert tonum.nplike is self._backend.index_nplike self._handle_error( @@ -630,7 +630,7 @@ def _num(self, axis, depth=0): ) return ak.contents.NumpyArray(tonum, parameters=None, backend=self._backend) else: - next = self._content._num(posaxis, depth + 1) + next = self._content._num(posaxis, depth_m1 + 1) return ak.contents.RegularArray( next, self._size, self._length, parameters=self._parameters ) diff --git a/src/awkward/contents/unionarray.py b/src/awkward/contents/unionarray.py index bf63cfdd60..23887b208b 100644 --- a/src/awkward/contents/unionarray.py +++ b/src/awkward/contents/unionarray.py @@ -759,9 +759,9 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth=0): + def _num(self, axis, depth_m1): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis == depth_m1: out = self.length if ak._util.is_integer(out): return np.int64(out) @@ -770,7 +770,7 @@ def _num(self, axis, depth=0): else: contents = [] for content in self._contents: - contents.append(content._num(posaxis, depth)) + contents.append(content._num(posaxis, depth_m1)) return UnionArray.simplified( self._tags, self._index, diff --git a/src/awkward/contents/unmaskedarray.py b/src/awkward/contents/unmaskedarray.py index d66d13a925..b2da5d371e 100644 --- a/src/awkward/contents/unmaskedarray.py +++ b/src/awkward/contents/unmaskedarray.py @@ -233,9 +233,9 @@ def project(self, mask=None): else: return self._content - def _num(self, axis, depth=0): + def _num(self, axis, depth_m1): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis == depth_m1: out = self.length if ak._util.is_integer(out): return np.int64(out) @@ -243,7 +243,7 @@ def _num(self, axis, depth=0): return out else: return ak.contents.UnmaskedArray( - self._content._num(posaxis, depth), parameters=self._parameters + self._content._num(posaxis, depth_m1), parameters=self._parameters ) def _offsets_and_flattened(self, axis, depth): diff --git a/tests/test_0163-negative-axis-wrap.py b/tests/test_0163-negative-axis-wrap.py index b30ac36f2e..a4b2fa645c 100644 --- a/tests/test_0163-negative-axis-wrap.py +++ b/tests/test_0163-negative-axis-wrap.py @@ -24,7 +24,7 @@ def test_array_3d(): ] with pytest.raises(ValueError) as err: assert ak.operations.num(array, axis=3) - assert "axis=3 exceeds the depth of this array (2)" in str(err.value) + assert "axis=3 exceeds the depth of this array" in str(err.value) assert to_list(ak.operations.num(array, axis=-1)) == [ [2, 2, 2, 2, 2], @@ -51,7 +51,7 @@ def test_list_array(): with pytest.raises(ValueError) as err: assert ak.operations.num(array, axis=3) - assert "axis=3 exceeds the depth of this array (2)" in str(err.value) + assert "axis=3 exceeds the depth of this array" in str(err.value) assert ak.operations.num(array, axis=-1).to_list() == [ [2, 2, 2, 2, 2], @@ -62,7 +62,7 @@ def test_list_array(): assert ak.operations.num(array, axis=-3) == 3 with pytest.raises(ValueError) as err: assert ak.operations.num(array, axis=-4) - assert "axis=-4 exceeds the depth of this array (3)" in str(err.value) + assert "axis=-4 exceeds the depth of this array" in str(err.value) def test_record_array(): @@ -82,7 +82,7 @@ def test_record_array(): ] with pytest.raises(ValueError) as err: assert ak.operations.num(array, axis=2) - assert "axis=2 exceeds the depth of this array (1)" in str(err.value) + assert "axis=2 exceeds the depth of this array" in str(err.value) assert ak.operations.num(array, axis=-1).to_list() == [ {"x": 1, "y": [0, 1]}, @@ -102,8 +102,8 @@ def test_record_array_axis_out_of_range(): with pytest.raises(ValueError) as err: assert ak.operations.num(array, axis=-2) - assert "axis=-2 exceeds the depth of this array (2)" in str(err.value) + assert "axis=-2 exceeds the depth of this array" in str(err.value) with pytest.raises(ValueError) as err: assert ak.operations.num(array, axis=-3) - assert "axis=-3 exceeds the depth (2) of this array" in str(err.value) + assert "axis=-3 exceeds the depth" in str(err.value) diff --git a/tests/test_1137-num.py b/tests/test_1137-num.py index d64d805668..175e19e50d 100644 --- a/tests/test_1137-num.py +++ b/tests/test_1137-num.py @@ -56,7 +56,7 @@ def test_numpyarray(): ] with pytest.raises(ValueError) as err: ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array (3)") + assert str(err.value).startswith("axis=4 exceeds the depth of this array") def test_regulararray(): @@ -73,7 +73,7 @@ def test_regulararray(): ] with pytest.raises(ValueError) as err: ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array (3)") + assert str(err.value).startswith("axis=4 exceeds the depth of this array") empty = ak.contents.RegularArray( ak.highlevel.Array([[1, 2, 3], [], [4, 5]]).layout, 0, zeros_length=0 @@ -110,7 +110,7 @@ def test_listarray(): ] with pytest.raises(ValueError) as err: ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array (3)") + assert str(err.value).startswith("axis=4 exceeds the depth of this array") def test_listoffsetarray(): @@ -138,7 +138,7 @@ def test_listoffsetarray(): ] with pytest.raises(ValueError) as err: ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array (3)") + assert str(err.value).startswith("axis=4 exceeds the depth of this array") def test_indexedarray(): @@ -171,7 +171,7 @@ def test_indexedarray(): with pytest.raises(ValueError) as err: ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array (3)") + assert str(err.value).startswith("axis=4 exceeds the depth of this array") def test_indexedoptionarray(): @@ -208,7 +208,7 @@ def test_indexedoptionarray(): with pytest.raises(ValueError) as err: ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array (3)") + assert str(err.value).startswith("axis=4 exceeds the depth of this array") def test_recordarray(): @@ -311,7 +311,7 @@ def test_array_3d(): ] with pytest.raises(ValueError) as err: assert ak._do.num(array, axis=3) - assert str(err.value).startswith("axis=3 exceeds the depth of this array (2)") + assert str(err.value).startswith("axis=3 exceeds the depth of this array") assert to_list(ak._do.num(array, axis=-1)) == [ [2, 2, 2, 2, 2], @@ -323,4 +323,4 @@ def test_array_3d(): with pytest.raises(ValueError) as err: assert ak._do.num(array, axis=-4) - assert str(err.value).startswith("axis=-4 exceeds the depth of this array (3)") + assert str(err.value).startswith("axis=-4 exceeds the depth of this array") From 6cbb2bc8cccdff7560befb4c35b554635d7bdb4a Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 02:06:47 -0600 Subject: [PATCH 15/26] Replace 'Content._num' with a 'recursively_apply' implementation and eliminate the weird axis=0 output for records. --- src/awkward/contents/bitmaskedarray.py | 3 - src/awkward/contents/bytemaskedarray.py | 18 --- src/awkward/contents/emptyarray.py | 14 -- src/awkward/contents/indexedarray.py | 11 -- src/awkward/contents/indexedoptionarray.py | 15 --- src/awkward/contents/listarray.py | 32 ----- src/awkward/contents/listoffsetarray.py | 36 ----- src/awkward/contents/numpyarray.py | 36 ----- src/awkward/contents/recordarray.py | 29 ----- src/awkward/contents/regulararray.py | 23 ---- src/awkward/contents/unionarray.py | 19 --- src/awkward/contents/unmaskedarray.py | 13 -- src/awkward/operations/ak_num.py | 33 ++++- tests/test_0163-negative-axis-wrap.py | 4 +- tests/test_1137-num.py | 145 +++++++++++---------- 15 files changed, 102 insertions(+), 329 deletions(-) diff --git a/src/awkward/contents/bitmaskedarray.py b/src/awkward/contents/bitmaskedarray.py index 0200df0aec..2cfde7adc4 100644 --- a/src/awkward/contents/bitmaskedarray.py +++ b/src/awkward/contents/bitmaskedarray.py @@ -434,9 +434,6 @@ def _getitem_next(self, head, tail, advanced): def project(self, mask=None): return self.to_ByteMaskedArray().project(mask) - def _num(self, axis, depth_m1): - return self.to_ByteMaskedArray()._num(axis, depth_m1) - def _offsets_and_flattened(self, axis, depth): return self.to_ByteMaskedArray._offsets_and_flattened(axis, depth) diff --git a/src/awkward/contents/bytemaskedarray.py b/src/awkward/contents/bytemaskedarray.py index 81d17a0475..1eb38a6578 100644 --- a/src/awkward/contents/bytemaskedarray.py +++ b/src/awkward/contents/bytemaskedarray.py @@ -556,24 +556,6 @@ def project(self, mask=None): return self._content._carry(nextcarry, False) - def _num(self, axis, depth_m1): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth_m1: - out = self.length - if ak._util.is_integer(out): - return np.int64(out) - else: - return out - else: - _, nextcarry, outindex = self._nextcarry_outindex(self._backend) - - next = self._content._carry(nextcarry, False) - out = next._num(posaxis, depth_m1) - - return ak.contents.IndexedOptionArray.simplified( - outindex, out, parameters=self._parameters - ) - def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) if posaxis == depth: diff --git a/src/awkward/contents/emptyarray.py b/src/awkward/contents/emptyarray.py index cd634fe86d..7b79b8b37e 100644 --- a/src/awkward/contents/emptyarray.py +++ b/src/awkward/contents/emptyarray.py @@ -158,20 +158,6 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth_m1): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - - if posaxis == depth_m1: - out = self.length - if ak._util.is_integer(out): - return np.int64(out) - else: - return out - else: - # TODO: This was changed to use `nplike` instead of `index_nplike`. Is this OK? - out = ak.index.Index64.empty(0, nplike=self._backend.nplike) - return ak.contents.NumpyArray(out, parameters=None, backend=self._backend) - def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) if posaxis == depth: diff --git a/src/awkward/contents/indexedarray.py b/src/awkward/contents/indexedarray.py index a2eb1d0273..93aa2fc2d1 100644 --- a/src/awkward/contents/indexedarray.py +++ b/src/awkward/contents/indexedarray.py @@ -388,17 +388,6 @@ def project(self, mask=None): ) ) - def _num(self, axis, depth_m1): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth_m1: - out = self.length - if ak._util.is_integer(out): - return np.int64(out) - else: - return out - else: - return self.project()._num(posaxis, depth_m1) - def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) if posaxis == depth: diff --git a/src/awkward/contents/indexedoptionarray.py b/src/awkward/contents/indexedoptionarray.py index 84f4321726..e4ab62aa23 100644 --- a/src/awkward/contents/indexedoptionarray.py +++ b/src/awkward/contents/indexedoptionarray.py @@ -475,21 +475,6 @@ def project(self, mask=None): return self._content._carry(nextcarry, False) - def _num(self, axis, depth_m1): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth_m1: - out = self.length - if ak._util.is_integer(out): - return np.int64(out) - else: - return out - _, nextcarry, outindex = self._nextcarry_outindex(self._backend) - next = self._content._carry(nextcarry, False) - out = next._num(posaxis, depth_m1) - return ak.contents.IndexedOptionArray.simplified( - outindex, out, parameters=self._parameters - ) - def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) if posaxis == depth: diff --git a/src/awkward/contents/listarray.py b/src/awkward/contents/listarray.py index d5ab7c9b83..fd6edd8468 100644 --- a/src/awkward/contents/listarray.py +++ b/src/awkward/contents/listarray.py @@ -894,38 +894,6 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth_m1): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth_m1: - out = self.length - if ak._util.is_integer(out): - return np.int64(out) - else: - return out - elif posaxis == depth_m1 + 1: - tonum = ak.index.Index64.empty(self.length, self._backend.index_nplike) - assert ( - tonum.nplike is self._backend.index_nplike - and self._starts.nplike is self._backend.index_nplike - and self._stops.nplike is self._backend.index_nplike - ) - self._handle_error( - self._backend[ - "awkward_ListArray_num", - tonum.dtype.type, - self._starts.dtype.type, - self._stops.dtype.type, - ]( - tonum.data, - self._starts.data, - self._stops.data, - self.length, - ) - ) - return ak.contents.NumpyArray(tonum, parameters=None, backend=self._backend) - else: - return self.to_ListOffsetArray64(True)._num(posaxis, depth_m1) - def _offsets_and_flattened(self, axis, depth): return self.to_ListOffsetArray64(True)._offsets_and_flattened(axis, depth) diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py index 120f805d74..020ba734f4 100644 --- a/src/awkward/contents/listoffsetarray.py +++ b/src/awkward/contents/listoffsetarray.py @@ -601,42 +601,6 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth_m1): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth_m1: - out = self.length - if ak._util.is_integer(out): - return np.int64(out) - else: - return out - elif posaxis == depth_m1 + 1: - tonum = ak.index.Index64.empty(self.length, self._backend.index_nplike) - assert ( - tonum.nplike is self._backend.index_nplike - and self.starts.nplike is self._backend.index_nplike - and self.stops.nplike is self._backend.index_nplike - ) - self._handle_error( - self._backend[ - "awkward_ListArray_num", - tonum.dtype.type, - self.starts.dtype.type, - self.stops.dtype.type, - ]( - tonum.data, - self.starts.data, - self.stops.data, - self.length, - ) - ) - return ak.contents.NumpyArray(tonum, parameters=None, backend=self._backend) - else: - next = self._content._num(posaxis, depth_m1 + 1) - offsets = self._compact_offsets64(True) - return ak.contents.ListOffsetArray( - offsets, next, parameters=self._parameters - ) - def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) if posaxis == depth: diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index 4d69c1ca68..8656213f72 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -315,42 +315,6 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth_m1): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth_m1: - out = self.length - if ak._util.is_integer(out): - return np.int64(out) - else: - return out - shape = [] - reps = 1 - size = self.length - i = 0 - while i < self._data.ndim - 1 and depth_m1 < posaxis: - shape.append(self.shape[i]) - reps *= self.shape[i] - size = self.shape[i + 1] - i += 1 - depth_m1 += 1 - if posaxis > depth_m1: - raise ak._errors.wrap_error( - np.AxisError(f"axis={axis} exceeds the depth of this array ({depth_m1 + 1})") - ) - - tonum = ak.index.Index64.empty(reps, self._backend.index_nplike) - assert tonum.nplike is self._backend.index_nplike - self._handle_error( - self._backend["awkward_RegularArray_num", tonum.dtype.type]( - tonum.data, size, reps - ) - ) - return ak.contents.NumpyArray( - tonum.data.reshape(shape), - parameters=self._parameters, - backend=self._backend, - ) - def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) if posaxis == depth: diff --git a/src/awkward/contents/recordarray.py b/src/awkward/contents/recordarray.py index 4878cf691b..18af1d415d 100644 --- a/src/awkward/contents/recordarray.py +++ b/src/awkward/contents/recordarray.py @@ -443,35 +443,6 @@ def _getitem_next(self, head, tail, advanced): ) return next._getitem_next(nexthead, nexttail, advanced) - def _num(self, axis, depth_m1): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth_m1: - npsingle = self._backend.index_nplike.full((1,), self.length, np.int64) - single = ak.index.Index64(npsingle, nplike=self._backend.index_nplike) - singleton = ak.contents.NumpyArray( - single, parameters=None, backend=self._backend - ) - contents = [singleton] * len(self._contents) - record = ak.contents.RecordArray( - contents, - self._fields, - 1, - parameters=self._parameters, - backend=self._backend, - ) - return record[0] - else: - contents = [] - for content in self._contents: - contents.append(content._num(posaxis, depth_m1)) - return ak.contents.RecordArray( - contents, - self._fields, - self._length, - parameters=self._parameters, - backend=self._backend, - ) - def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) if posaxis == depth: diff --git a/src/awkward/contents/regulararray.py b/src/awkward/contents/regulararray.py index 4bfd30f2e5..6b78684b43 100644 --- a/src/awkward/contents/regulararray.py +++ b/src/awkward/contents/regulararray.py @@ -612,29 +612,6 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth_m1): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth_m1: - out = self._length - if ak._util.is_integer(out): - return np.int64(out) - else: - return out - elif posaxis == depth_m1 + 1: - tonum = ak.index.Index64.empty(self._length, self._backend.index_nplike) - assert tonum.nplike is self._backend.index_nplike - self._handle_error( - self._backend["awkward_RegularArray_num", tonum.dtype.type]( - tonum.data, self._size, self._length - ) - ) - return ak.contents.NumpyArray(tonum, parameters=None, backend=self._backend) - else: - next = self._content._num(posaxis, depth_m1 + 1) - return ak.contents.RegularArray( - next, self._size, self._length, parameters=self._parameters - ) - def _offsets_and_flattened(self, axis, depth): return self.to_ListOffsetArray64(True)._offsets_and_flattened(axis, depth) diff --git a/src/awkward/contents/unionarray.py b/src/awkward/contents/unionarray.py index 23887b208b..936d62d3ba 100644 --- a/src/awkward/contents/unionarray.py +++ b/src/awkward/contents/unionarray.py @@ -759,25 +759,6 @@ def _getitem_next(self, head, tail, advanced): else: raise ak._errors.wrap_error(AssertionError(repr(head))) - def _num(self, axis, depth_m1): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth_m1: - out = self.length - if ak._util.is_integer(out): - return np.int64(out) - else: - return out - else: - contents = [] - for content in self._contents: - contents.append(content._num(posaxis, depth_m1)) - return UnionArray.simplified( - self._tags, - self._index, - contents, - parameters=self._parameters, - ) - def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) diff --git a/src/awkward/contents/unmaskedarray.py b/src/awkward/contents/unmaskedarray.py index b2da5d371e..8e33e60f8a 100644 --- a/src/awkward/contents/unmaskedarray.py +++ b/src/awkward/contents/unmaskedarray.py @@ -233,19 +233,6 @@ def project(self, mask=None): else: return self._content - def _num(self, axis, depth_m1): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth_m1: - out = self.length - if ak._util.is_integer(out): - return np.int64(out) - else: - return out - else: - return ak.contents.UnmaskedArray( - self._content._num(posaxis, depth_m1), parameters=self._parameters - ) - def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) if posaxis == depth: diff --git a/src/awkward/operations/ak_num.py b/src/awkward/operations/ak_num.py index 40f7063eca..5428628ac3 100644 --- a/src/awkward/operations/ak_num.py +++ b/src/awkward/operations/ak_num.py @@ -72,10 +72,31 @@ def num(array, axis=1, *, highlevel=True, behavior=None): def _impl(array, axis, highlevel, behavior): - layout = ak.operations.to_layout(array, allow_record=False, allow_other=False) + layout = ak.operations.to_layout(array) behavior = ak._util.behavior_of(array, behavior=behavior) - out = ak._do.num(layout, axis=axis) - if isinstance(out, (ak.contents.Content, ak.record.Record)): - return ak._util.wrap(out, behavior, highlevel) - else: - return out + + if not ak._util.is_integer(axis): + raise ak._errors.wrap_error( + TypeError(f"'axis' must be an integer, not {axis!r}") + ) + + if ak._do.maybe_posaxis(layout, axis, 1) == 0: + if isinstance(layout, ak.record.Record): + return 1 + else: + return layout.length + + def action(layout, depth, **kwargs): + posaxis = ak._do.maybe_posaxis(layout, axis, depth) + + if posaxis == depth and layout.is_list: + return ak.contents.NumpyArray(layout.stops.data - layout.starts.data) + + elif layout.is_leaf: + raise ak._errors.wrap_error( + np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") + ) + + out = ak._do.recursively_apply(layout, action, behavior, numpy_to_regular=True) + + return ak._util.wrap(out, behavior, highlevel) diff --git a/tests/test_0163-negative-axis-wrap.py b/tests/test_0163-negative-axis-wrap.py index a4b2fa645c..d2d0437d9d 100644 --- a/tests/test_0163-negative-axis-wrap.py +++ b/tests/test_0163-negative-axis-wrap.py @@ -36,7 +36,7 @@ def test_array_3d(): with pytest.raises(ValueError) as err: assert ak.operations.num(array, axis=-4) - assert "axis=-4 exceeds the depth (3) of this array" in str(err.value) + assert "axis=-4 exceeds the depth" in str(err.value) def test_list_array(): @@ -74,7 +74,7 @@ def test_record_array(): ] ) - assert ak.operations.num(array, axis=0).to_list() == {"x": 3, "y": 3} + assert ak.operations.num(array, axis=0) == 3 assert ak.operations.num(array, axis=1).to_list() == [ {"x": 1, "y": 2}, {"x": 2, "y": 3}, diff --git a/tests/test_1137-num.py b/tests/test_1137-num.py index 175e19e50d..809485bbf9 100644 --- a/tests/test_1137-num.py +++ b/tests/test_1137-num.py @@ -29,34 +29,35 @@ def test_bytemaskedarray_num(): None, [[], [10.0, 11.1, 12.2]], ] - assert ak._do.num(array, axis=0) == 5 - assert ak._do.num(array, axis=-3) == 5 - assert to_list(ak._do.num(array, axis=1)) == [3, 0, None, None, 2] - assert to_list(ak._do.num(array, axis=-2)) == [3, 0, None, None, 2] - assert to_list(ak._do.num(array, axis=2)) == [[3, 0, 2], [], None, None, [0, 3]] - assert to_list(ak._do.num(array, axis=-1)) == [[3, 0, 2], [], None, None, [0, 3]] + assert ak.num(array, axis=0) == 5 + assert ak.num(array, axis=-3) == 5 + assert to_list(ak.num(array, axis=1)) == [3, 0, None, None, 2] + assert to_list(ak.num(array, axis=-2)) == [3, 0, None, None, 2] + assert to_list(ak.num(array, axis=2)) == [[3, 0, 2], [], None, None, [0, 3]] + assert to_list(ak.num(array, axis=-1)) == [[3, 0, 2], [], None, None, [0, 3]] def test_emptyarray(): array = ak.contents.EmptyArray() - assert to_list(ak._do.num(array, 0)) == 0 - assert to_list(ak._do.num(array, 1)) == [] - assert to_list(ak._do.num(array, 2)) == [] + assert to_list(ak.num(array, 0)) == 0 + with pytest.raises(np.AxisError) as err: + ak.num(array, 1) + assert "axis=1 exceeds the depth" in str(err.value) def test_numpyarray(): array = ak.contents.NumpyArray(np.arange(2 * 3 * 5 * 7).reshape(2, 3, 5, 7)) - assert ak._do.num(array, 0) == 2 - assert to_list(ak._do.num(array, 1)) == [3, 3] - assert to_list(ak._do.num(array, axis=2)) == [[5, 5, 5], [5, 5, 5]] - assert to_list(ak._do.num(array, 3)) == [ + assert ak.num(array, 0) == 2 + assert to_list(ak.num(array, 1)) == [3, 3] + assert to_list(ak.num(array, axis=2)) == [[5, 5, 5], [5, 5, 5]] + assert to_list(ak.num(array, 3)) == [ [[7, 7, 7, 7, 7], [7, 7, 7, 7, 7], [7, 7, 7, 7, 7]], [[7, 7, 7, 7, 7], [7, 7, 7, 7, 7], [7, 7, 7, 7, 7]], ] - with pytest.raises(ValueError) as err: - ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array") + with pytest.raises(np.AxisError) as err: + ak.num(array, 4) + assert "axis=4 exceeds the depth" in str(err.value) def test_regulararray(): @@ -64,24 +65,24 @@ def test_regulararray(): np.arange(2 * 3 * 5 * 7).reshape(2, 3, 5, 7) ).to_RegularArray() - assert ak._do.num(array, 0) == 2 - assert to_list(ak._do.num(array, 1)) == [3, 3] - assert to_list(ak._do.num(array, 2)) == [[5, 5, 5], [5, 5, 5]] - assert to_list(ak._do.num(array, 3)) == [ + assert ak.num(array, 0) == 2 + assert to_list(ak.num(array, 1)) == [3, 3] + assert to_list(ak.num(array, 2)) == [[5, 5, 5], [5, 5, 5]] + assert to_list(ak.num(array, 3)) == [ [[7, 7, 7, 7, 7], [7, 7, 7, 7, 7], [7, 7, 7, 7, 7]], [[7, 7, 7, 7, 7], [7, 7, 7, 7, 7], [7, 7, 7, 7, 7]], ] - with pytest.raises(ValueError) as err: - ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array") + with pytest.raises(np.AxisError) as err: + ak.num(array, 4) + assert "axis=4 exceeds the depth" in str(err.value) empty = ak.contents.RegularArray( ak.highlevel.Array([[1, 2, 3], [], [4, 5]]).layout, 0, zeros_length=0 ) - assert ak._do.num(empty, axis=0) == 0 - assert to_list(ak._do.num(empty, axis=1)) == [] - assert to_list(ak._do.num(empty, axis=2)) == [] + assert ak.num(empty, axis=0) == 0 + assert to_list(ak.num(empty, axis=1)) == [] + assert to_list(ak.num(empty, axis=2)) == [] def test_listarray(): @@ -100,17 +101,17 @@ def test_listarray(): [[[18, 19], [20, 21], [22, 23]], [[24, 25], [26, 27], [28, 29]]], ] - assert to_list(ak._do.num(array, 0)) == 3 - assert to_list(ak._do.num(array, 1)) == [3, 0, 2] - assert to_list(ak._do.num(array, 2)) == [[3, 3, 3], [], [3, 3]] - assert to_list(ak._do.num(array, 3)) == [ + assert to_list(ak.num(array, 0)) == 3 + assert to_list(ak.num(array, 1)) == [3, 0, 2] + assert to_list(ak.num(array, 2)) == [[3, 3, 3], [], [3, 3]] + assert to_list(ak.num(array, 3)) == [ [[2, 2, 2], [2, 2, 2], [2, 2, 2]], [], [[2, 2, 2], [2, 2, 2]], ] - with pytest.raises(ValueError) as err: - ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array") + with pytest.raises(np.AxisError) as err: + ak.num(array, 4) + assert "axis=4 exceeds the depth" in str(err.value) def test_listoffsetarray(): @@ -128,17 +129,17 @@ def test_listoffsetarray(): [[[18, 19], [20, 21], [22, 23]], [[24, 25], [26, 27], [28, 29]]], ] - assert to_list(ak._do.num(array, 0)) == 3 - assert to_list(ak._do.num(array, 1)) == [3, 0, 2] - assert to_list(ak._do.num(array, 2)) == [[3, 3, 3], [], [3, 3]] - assert to_list(ak._do.num(array, 3)) == [ + assert to_list(ak.num(array, 0)) == 3 + assert to_list(ak.num(array, 1)) == [3, 0, 2] + assert to_list(ak.num(array, 2)) == [[3, 3, 3], [], [3, 3]] + assert to_list(ak.num(array, 3)) == [ [[2, 2, 2], [2, 2, 2], [2, 2, 2]], [], [[2, 2, 2], [2, 2, 2]], ] - with pytest.raises(ValueError) as err: - ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array") + with pytest.raises(np.AxisError) as err: + ak.num(array, 4) + assert "axis=4 exceeds the depth" in str(err.value) def test_indexedarray(): @@ -159,19 +160,19 @@ def test_indexedarray(): ], ] - assert to_list(ak._do.num(array, 0)) == 4 - assert to_list(ak._do.num(array, 1)) == [2, 2, 0, 3] - assert to_list(ak._do.num(array, 2)) == [[3, 3], [3, 3], [], [3, 3, 3]] - assert to_list(ak._do.num(array, 3)) == [ + assert to_list(ak.num(array, 0)) == 4 + assert to_list(ak.num(array, 1)) == [2, 2, 0, 3] + assert to_list(ak.num(array, 2)) == [[3, 3], [3, 3], [], [3, 3, 3]] + assert to_list(ak.num(array, 3)) == [ [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [], [[2, 2, 2], [2, 2, 2], [2, 2, 2]], ] - with pytest.raises(ValueError) as err: - ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array") + with pytest.raises(np.AxisError) as err: + ak.num(array, 4) + assert "axis=4 exceeds the depth" in str(err.value) def test_indexedoptionarray(): @@ -194,10 +195,10 @@ def test_indexedoptionarray(): ], ] - assert to_list(ak._do.num(array, 0)) == 6 - assert to_list(ak._do.num(array, 1)) == [2, None, 2, 0, None, 3] - assert to_list(ak._do.num(array, 2)) == [[3, 3], None, [3, 3], [], None, [3, 3, 3]] - assert to_list(ak._do.num(array, 3)) == [ + assert to_list(ak.num(array, 0)) == 6 + assert to_list(ak.num(array, 1)) == [2, None, 2, 0, None, 3] + assert to_list(ak.num(array, 2)) == [[3, 3], None, [3, 3], [], None, [3, 3, 3]] + assert to_list(ak.num(array, 3)) == [ [[2, 2, 2], [2, 2, 2]], None, [[2, 2, 2], [2, 2, 2]], @@ -206,9 +207,9 @@ def test_indexedoptionarray(): [[2, 2, 2], [2, 2, 2], [2, 2, 2]], ] - with pytest.raises(ValueError) as err: - ak._do.num(array, 4) - assert str(err.value).startswith("axis=4 exceeds the depth of this array") + with pytest.raises(np.AxisError) as err: + ak.num(array, 4) + assert "axis=4 exceeds the depth" in str(err.value) def test_recordarray(): @@ -222,7 +223,7 @@ def test_recordarray(): highlevel=False, ) - assert to_list(ak._do.num(array, 0)) == {"x": 4, "y": 4} + assert ak.num(array, 0) == 4 array = ak.operations.from_iter( [ @@ -234,14 +235,14 @@ def test_recordarray(): highlevel=False, ) - assert to_list(ak._do.num(array, 0)) == {"x": 4, "y": 4} - assert to_list(ak._do.num(array, 1)) == [ + assert ak.num(array, 0) == 4 + assert to_list(ak.num(array, 1)) == [ {"x": 3, "y": 0}, {"x": 2, "y": 1}, {"x": 1, "y": 2}, {"x": 0, "y": 3}, ] - assert to_list(ak._do.num(array, 1)[2]) == {"x": 1, "y": 2} + assert to_list(ak.num(array, 1)[2]) == {"x": 1, "y": 2} array = ak.operations.from_iter( [ @@ -253,14 +254,14 @@ def test_recordarray(): highlevel=False, ) - assert to_list(ak._do.num(array, 0)) == {"x": 4, "y": 4} - assert to_list(ak._do.num(array, 1)) == [ + assert ak.num(array, 0) == 4 + assert to_list(ak.num(array, 1)) == [ {"x": 1, "y": 0}, {"x": 1, "y": 1}, {"x": 1, "y": 2}, {"x": 1, "y": 3}, ] - assert to_list(ak._do.num(array, 1)[2]) == {"x": 1, "y": 2} + assert to_list(ak.num(array, 1)[2]) == {"x": 1, "y": 2} def test_unionarray(): @@ -284,9 +285,9 @@ def test_unionarray(): [], ] - assert ak._do.num(array, 0) == 8 - assert isinstance(ak._do.num(array, 1), ak.contents.NumpyArray) - assert to_list(ak._do.num(array, 1)) == [0, 3, 1, 2, 2, 1, 3, 0] + assert ak.num(array, 0) == 8 + assert isinstance(ak.num(array, 1, highlevel=False), ak.contents.NumpyArray) + assert to_list(ak.num(array, 1)) == [0, 3, 1, 2, 2, 1, 3, 0] def test_highlevel(): @@ -302,25 +303,25 @@ def test_array_3d(): [[10, 11], [12, 13], [14, 15], [16, 17], [18, 19]], [[20, 21], [22, 23], [24, 25], [26, 27], [28, 29]], ] - assert ak._do.num(array, axis=0) == 3 - assert to_list(ak._do.num(array, axis=1)) == [5, 5, 5] - assert to_list(ak._do.num(array, axis=2)) == [ + assert ak.num(array, axis=0) == 3 + assert to_list(ak.num(array, axis=1)) == [5, 5, 5] + assert to_list(ak.num(array, axis=2)) == [ [2, 2, 2, 2, 2], [2, 2, 2, 2, 2], [2, 2, 2, 2, 2], ] with pytest.raises(ValueError) as err: - assert ak._do.num(array, axis=3) + assert ak.num(array, axis=3) assert str(err.value).startswith("axis=3 exceeds the depth of this array") - assert to_list(ak._do.num(array, axis=-1)) == [ + assert to_list(ak.num(array, axis=-1)) == [ [2, 2, 2, 2, 2], [2, 2, 2, 2, 2], [2, 2, 2, 2, 2], ] - assert to_list(ak._do.num(array, axis=-2)) == [5, 5, 5] - assert ak._do.num(array, axis=-3) == 3 + assert to_list(ak.num(array, axis=-2)) == [5, 5, 5] + assert ak.num(array, axis=-3) == 3 with pytest.raises(ValueError) as err: - assert ak._do.num(array, axis=-4) + assert ak.num(array, axis=-4) assert str(err.value).startswith("axis=-4 exceeds the depth of this array") From 3b2aadc5e7bad689661ba1e5543dcc0e2e0b6ac7 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 02:17:55 -0600 Subject: [PATCH 16/26] Shift Content._offsets_and_flattened to use the standard 'depth' definition. --- src/awkward/_do.py | 4 ++-- src/awkward/contents/bytemaskedarray.py | 2 +- src/awkward/contents/emptyarray.py | 2 +- src/awkward/contents/indexedarray.py | 2 +- src/awkward/contents/indexedoptionarray.py | 2 +- src/awkward/contents/listoffsetarray.py | 4 ++-- src/awkward/contents/numpyarray.py | 2 +- src/awkward/contents/recordarray.py | 4 ++-- src/awkward/contents/unionarray.py | 2 +- src/awkward/contents/unmaskedarray.py | 2 +- src/awkward/operations/ak_concatenate.py | 2 +- src/awkward/operations/ak_to_dataframe.py | 2 +- 12 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/awkward/_do.py b/src/awkward/_do.py index 4035faecfc..d60eed2f8b 100644 --- a/src/awkward/_do.py +++ b/src/awkward/_do.py @@ -305,8 +305,8 @@ def completely_flatten( return tuple(arrays) -def flatten(layout: Content, axis: Integral = 1, depth: Integral = 0) -> Content: - offsets, flattened = layout._offsets_and_flattened(axis, depth) +def flatten(layout: Content, axis: Integral = 1) -> Content: + offsets, flattened = layout._offsets_and_flattened(axis, 1) return flattened diff --git a/src/awkward/contents/bytemaskedarray.py b/src/awkward/contents/bytemaskedarray.py index 1eb38a6578..370a740e7e 100644 --- a/src/awkward/contents/bytemaskedarray.py +++ b/src/awkward/contents/bytemaskedarray.py @@ -558,7 +558,7 @@ def project(self, mask=None): def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: numnull, nextcarry, outindex = self._nextcarry_outindex(self._backend) diff --git a/src/awkward/contents/emptyarray.py b/src/awkward/contents/emptyarray.py index 7b79b8b37e..ee9ac893ed 100644 --- a/src/awkward/contents/emptyarray.py +++ b/src/awkward/contents/emptyarray.py @@ -160,7 +160,7 @@ def _getitem_next(self, head, tail, advanced): def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: raise ak._errors.wrap_error( np.AxisError(self, "axis=0 not allowed for flatten") ) diff --git a/src/awkward/contents/indexedarray.py b/src/awkward/contents/indexedarray.py index 93aa2fc2d1..d9e25de03e 100644 --- a/src/awkward/contents/indexedarray.py +++ b/src/awkward/contents/indexedarray.py @@ -390,7 +390,7 @@ def project(self, mask=None): def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: diff --git a/src/awkward/contents/indexedoptionarray.py b/src/awkward/contents/indexedoptionarray.py index e4ab62aa23..d0e1d59c38 100644 --- a/src/awkward/contents/indexedoptionarray.py +++ b/src/awkward/contents/indexedoptionarray.py @@ -477,7 +477,7 @@ def project(self, mask=None): def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: numnull, nextcarry, outindex = self._nextcarry_outindex(self._backend) diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py index 020ba734f4..bd43e2e1dc 100644 --- a/src/awkward/contents/listoffsetarray.py +++ b/src/awkward/contents/listoffsetarray.py @@ -603,10 +603,10 @@ def _getitem_next(self, head, tail, advanced): def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: listoffsetarray = self.to_ListOffsetArray64(True) stop = listoffsetarray.offsets[-1] content = listoffsetarray.content._getitem_range(slice(0, stop)) diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index 8656213f72..9124abe73a 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -317,7 +317,7 @@ def _getitem_next(self, head, tail, advanced): def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) elif len(self.shape) != 1: diff --git a/src/awkward/contents/recordarray.py b/src/awkward/contents/recordarray.py index 18af1d415d..7aea61642e 100644 --- a/src/awkward/contents/recordarray.py +++ b/src/awkward/contents/recordarray.py @@ -445,10 +445,10 @@ def _getitem_next(self, head, tail, advanced): def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: raise ak._errors.wrap_error( ValueError( "arrays of records cannot be flattened (but their contents can be; try a different 'axis')" diff --git a/src/awkward/contents/unionarray.py b/src/awkward/contents/unionarray.py index 936d62d3ba..45f51b98d1 100644 --- a/src/awkward/contents/unionarray.py +++ b/src/awkward/contents/unionarray.py @@ -762,7 +762,7 @@ def _getitem_next(self, head, tail, advanced): def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: diff --git a/src/awkward/contents/unmaskedarray.py b/src/awkward/contents/unmaskedarray.py index 8e33e60f8a..6590d0058a 100644 --- a/src/awkward/contents/unmaskedarray.py +++ b/src/awkward/contents/unmaskedarray.py @@ -235,7 +235,7 @@ def project(self, mask=None): def _offsets_and_flattened(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: offsets, flattened = self._content._offsets_and_flattened(posaxis, depth) diff --git a/src/awkward/operations/ak_concatenate.py b/src/awkward/operations/ak_concatenate.py index 465f9ba52e..915d331f29 100644 --- a/src/awkward/operations/ak_concatenate.py +++ b/src/awkward/operations/ak_concatenate.py @@ -226,7 +226,7 @@ def action(inputs, depth, **kwargs): all_flatten = [] for x in nextinputs: - o, f = x._offsets_and_flattened(1, 0) + o, f = x._offsets_and_flattened(1, 1) o = backend.index_nplike.asarray(o) c = o[1:] - o[:-1] backend.index_nplike.add(counts, c, out=counts) diff --git a/src/awkward/operations/ak_to_dataframe.py b/src/awkward/operations/ak_to_dataframe.py index 961f3ba76f..ee9bb5d4b6 100644 --- a/src/awkward/operations/ak_to_dataframe.py +++ b/src/awkward/operations/ak_to_dataframe.py @@ -167,7 +167,7 @@ def recurse(layout, row_arrays, col_names): return [(ak.operations.to_numpy(layout), row_arrays, col_names)] elif layout.purelist_depth > 1: - offsets, flattened = layout._offsets_and_flattened(axis=1, depth=0) + offsets, flattened = layout._offsets_and_flattened(axis=1, depth=1) offsets = numpy.asarray(offsets) starts, stops = offsets[:-1], offsets[1:] counts = stops - starts From 6d0beb824f83ad17742111bb098a4517a5ac9235 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 02:25:12 -0600 Subject: [PATCH 17/26] Content._offsets_and_flattened now uses maybe_posaxis. --- src/awkward/contents/bytemaskedarray.py | 6 +++--- src/awkward/contents/emptyarray.py | 4 ++-- src/awkward/contents/indexedarray.py | 6 +++--- src/awkward/contents/indexedoptionarray.py | 6 +++--- src/awkward/contents/listoffsetarray.py | 8 ++++---- src/awkward/contents/numpyarray.py | 6 +++--- src/awkward/contents/recordarray.py | 8 ++++---- src/awkward/contents/unionarray.py | 6 +++--- src/awkward/contents/unmaskedarray.py | 6 +++--- 9 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/awkward/contents/bytemaskedarray.py b/src/awkward/contents/bytemaskedarray.py index 370a740e7e..59bf81f2cf 100644 --- a/src/awkward/contents/bytemaskedarray.py +++ b/src/awkward/contents/bytemaskedarray.py @@ -557,15 +557,15 @@ def project(self, mask=None): return self._content._carry(nextcarry, False) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: numnull, nextcarry, outindex = self._nextcarry_outindex(self._backend) next = self._content._carry(nextcarry, False) - offsets, flattened = next._offsets_and_flattened(posaxis, depth) + offsets, flattened = next._offsets_and_flattened(axis, depth) if offsets.length == 0: return ( diff --git a/src/awkward/contents/emptyarray.py b/src/awkward/contents/emptyarray.py index ee9ac893ed..45fb779a25 100644 --- a/src/awkward/contents/emptyarray.py +++ b/src/awkward/contents/emptyarray.py @@ -159,8 +159,8 @@ def _getitem_next(self, head, tail, advanced): raise ak._errors.wrap_error(AssertionError(repr(head))) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error( np.AxisError(self, "axis=0 not allowed for flatten") ) diff --git a/src/awkward/contents/indexedarray.py b/src/awkward/contents/indexedarray.py index d9e25de03e..c2dc1a7900 100644 --- a/src/awkward/contents/indexedarray.py +++ b/src/awkward/contents/indexedarray.py @@ -389,12 +389,12 @@ def project(self, mask=None): ) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: - return self.project()._offsets_and_flattened(posaxis, depth) + return self.project()._offsets_and_flattened(axis, depth) def _mergeable_next(self, other, mergebool): if isinstance( diff --git a/src/awkward/contents/indexedoptionarray.py b/src/awkward/contents/indexedoptionarray.py index d0e1d59c38..b824e4f072 100644 --- a/src/awkward/contents/indexedoptionarray.py +++ b/src/awkward/contents/indexedoptionarray.py @@ -476,14 +476,14 @@ def project(self, mask=None): return self._content._carry(nextcarry, False) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: numnull, nextcarry, outindex = self._nextcarry_outindex(self._backend) next = self._content._carry(nextcarry, False) - offsets, flattened = next._offsets_and_flattened(posaxis, depth) + offsets, flattened = next._offsets_and_flattened(axis, depth) if offsets.length == 0: return ( diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py index bd43e2e1dc..f762babfbb 100644 --- a/src/awkward/contents/listoffsetarray.py +++ b/src/awkward/contents/listoffsetarray.py @@ -602,11 +602,11 @@ def _getitem_next(self, head, tail, advanced): raise ak._errors.wrap_error(AssertionError(repr(head))) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) - elif posaxis + 1 == depth + 1: + elif posaxis is not None and posaxis + 1 == depth + 1: listoffsetarray = self.to_ListOffsetArray64(True) stop = listoffsetarray.offsets[-1] content = listoffsetarray.content._getitem_range(slice(0, stop)) @@ -614,7 +614,7 @@ def _offsets_and_flattened(self, axis, depth): else: inneroffsets, flattened = self._content._offsets_and_flattened( - posaxis, depth + 1 + axis, depth + 1 ) offsets = ak.index.Index64.zeros( 0, diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index 9124abe73a..f0c0e24438 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -316,12 +316,12 @@ def _getitem_next(self, head, tail, advanced): raise ak._errors.wrap_error(AssertionError(repr(head))) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) elif len(self.shape) != 1: - return self.to_RegularArray()._offsets_and_flattened(posaxis, depth) + return self.to_RegularArray()._offsets_and_flattened(axis, depth) else: raise ak._errors.wrap_error( diff --git a/src/awkward/contents/recordarray.py b/src/awkward/contents/recordarray.py index 7aea61642e..4e77118270 100644 --- a/src/awkward/contents/recordarray.py +++ b/src/awkward/contents/recordarray.py @@ -444,11 +444,11 @@ def _getitem_next(self, head, tail, advanced): return next._getitem_next(nexthead, nexttail, advanced) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) - elif posaxis + 1 == depth + 1: + elif posaxis is not None and posaxis + 1 == depth + 1: raise ak._errors.wrap_error( ValueError( "arrays of records cannot be flattened (but their contents can be; try a different 'axis')" @@ -459,7 +459,7 @@ def _offsets_and_flattened(self, axis, depth): contents = [] for content in self._contents: trimmed = content._getitem_range(slice(0, self.length)) - offsets, flattened = trimmed._offsets_and_flattened(posaxis, depth) + offsets, flattened = trimmed._offsets_and_flattened(axis, depth) if self._backend.nplike.known_shape and offsets.length != 0: raise ak._errors.wrap_error( AssertionError( diff --git a/src/awkward/contents/unionarray.py b/src/awkward/contents/unionarray.py index 45f51b98d1..cd5ce5ef52 100644 --- a/src/awkward/contents/unionarray.py +++ b/src/awkward/contents/unionarray.py @@ -760,9 +760,9 @@ def _getitem_next(self, head, tail, advanced): raise ak._errors.wrap_error(AssertionError(repr(head))) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) + posaxis = ak._do.maybe_posaxis(self, axis, depth) - if posaxis + 1 == depth: + if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: @@ -774,7 +774,7 @@ def _offsets_and_flattened(self, axis, depth): for i in range(len(self._contents)): offsets, flattened = self._contents[i]._offsets_and_flattened( - posaxis, depth + axis, depth ) offsetsraws[i] = offsets.ptr contents.append(flattened) diff --git a/src/awkward/contents/unmaskedarray.py b/src/awkward/contents/unmaskedarray.py index 6590d0058a..e735abae9f 100644 --- a/src/awkward/contents/unmaskedarray.py +++ b/src/awkward/contents/unmaskedarray.py @@ -234,11 +234,11 @@ def project(self, mask=None): return self._content def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: - offsets, flattened = self._content._offsets_and_flattened(posaxis, depth) + offsets, flattened = self._content._offsets_and_flattened(axis, depth) if offsets.length == 0: return ( offsets, From a2b812017548ba90ae6dfc52ec9b956cec3ba32a Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 02:30:00 -0600 Subject: [PATCH 18/26] Shift Content._local_index to use the standard 'depth' definition. --- src/awkward/_do.py | 2 +- src/awkward/contents/bytemaskedarray.py | 2 +- src/awkward/contents/indexedarray.py | 2 +- src/awkward/contents/indexedoptionarray.py | 2 +- src/awkward/contents/listarray.py | 4 ++-- src/awkward/contents/listoffsetarray.py | 4 ++-- src/awkward/contents/numpyarray.py | 2 +- src/awkward/contents/recordarray.py | 2 +- src/awkward/contents/regulararray.py | 4 ++-- src/awkward/contents/unionarray.py | 2 +- src/awkward/contents/unmaskedarray.py | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/awkward/_do.py b/src/awkward/_do.py index d60eed2f8b..aba7fc39de 100644 --- a/src/awkward/_do.py +++ b/src/awkward/_do.py @@ -185,7 +185,7 @@ def axis_wrap_if_negative( def local_index(layout: Content, axis: Integral): - return layout._local_index(axis, 0) + return layout._local_index(axis, 1) def combinations( diff --git a/src/awkward/contents/bytemaskedarray.py b/src/awkward/contents/bytemaskedarray.py index 59bf81f2cf..67eedb7b48 100644 --- a/src/awkward/contents/bytemaskedarray.py +++ b/src/awkward/contents/bytemaskedarray.py @@ -655,7 +655,7 @@ def _fill_none(self, value: Content) -> Content: def _local_index(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._local_index_axis0() else: _, nextcarry, outindex = self._nextcarry_outindex(self._backend) diff --git a/src/awkward/contents/indexedarray.py b/src/awkward/contents/indexedarray.py index c2dc1a7900..62ea48ceab 100644 --- a/src/awkward/contents/indexedarray.py +++ b/src/awkward/contents/indexedarray.py @@ -592,7 +592,7 @@ def _fill_none(self, value: Content) -> Content: def _local_index(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._local_index_axis0() else: return self.project()._local_index(posaxis, depth) diff --git a/src/awkward/contents/indexedoptionarray.py b/src/awkward/contents/indexedoptionarray.py index b824e4f072..5821961f94 100644 --- a/src/awkward/contents/indexedoptionarray.py +++ b/src/awkward/contents/indexedoptionarray.py @@ -741,7 +741,7 @@ def _fill_none(self, value: Content) -> Content: def _local_index(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._local_index_axis0() else: _, nextcarry, outindex = self._nextcarry_outindex(self._backend) diff --git a/src/awkward/contents/listarray.py b/src/awkward/contents/listarray.py index fd6edd8468..bd023d47bf 100644 --- a/src/awkward/contents/listarray.py +++ b/src/awkward/contents/listarray.py @@ -1069,9 +1069,9 @@ def _fill_none(self, value: Content) -> Content: def _local_index(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._local_index_axis0() - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: offsets = self._compact_offsets64(True) if self._backend.nplike.known_data: innerlength = offsets[offsets.length - 1] diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py index f762babfbb..5efbb4a672 100644 --- a/src/awkward/contents/listoffsetarray.py +++ b/src/awkward/contents/listoffsetarray.py @@ -716,9 +716,9 @@ def _fill_none(self, value: Content) -> Content: def _local_index(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._local_index_axis0() - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: offsets = self._compact_offsets64(True) if self._backend.nplike.known_data: innerlength = offsets[offsets.length - 1] diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index f0c0e24438..b0342f3482 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -435,7 +435,7 @@ def _fill_none(self, value: Content) -> Content: def _local_index(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._local_index_axis0() elif len(self.shape) <= 1: raise ak._errors.wrap_error( diff --git a/src/awkward/contents/recordarray.py b/src/awkward/contents/recordarray.py index 4e77118270..27b2b117ae 100644 --- a/src/awkward/contents/recordarray.py +++ b/src/awkward/contents/recordarray.py @@ -659,7 +659,7 @@ def _fill_none(self, value: Content) -> Content: def _local_index(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._local_index_axis0() else: contents = [] diff --git a/src/awkward/contents/regulararray.py b/src/awkward/contents/regulararray.py index 6b78684b43..63b6c44143 100644 --- a/src/awkward/contents/regulararray.py +++ b/src/awkward/contents/regulararray.py @@ -688,9 +688,9 @@ def _fill_none(self, value: Content) -> Content: def _local_index(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._local_index_axis0() - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: localindex = ak.index.Index64.empty( self._length * self._size, nplike=self._backend.index_nplike ) diff --git a/src/awkward/contents/unionarray.py b/src/awkward/contents/unionarray.py index cd5ce5ef52..0133c6fe12 100644 --- a/src/awkward/contents/unionarray.py +++ b/src/awkward/contents/unionarray.py @@ -1114,7 +1114,7 @@ def _fill_none(self, value: Content) -> Content: def _local_index(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._local_index_axis0() else: contents = [] diff --git a/src/awkward/contents/unmaskedarray.py b/src/awkward/contents/unmaskedarray.py index e735abae9f..a98da564c9 100644 --- a/src/awkward/contents/unmaskedarray.py +++ b/src/awkward/contents/unmaskedarray.py @@ -290,7 +290,7 @@ def _fill_none(self, value: Content) -> Content: def _local_index(self, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._local_index_axis0() else: return UnmaskedArray( From a00073f875002ad72fcb3c424ee9d8ffa48cb5af Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 02:39:47 -0600 Subject: [PATCH 19/26] Content._local_index now uses maybe_posaxis. --- src/awkward/contents/bytemaskedarray.py | 6 +++--- src/awkward/contents/emptyarray.py | 16 +++++++++++----- src/awkward/contents/indexedarray.py | 6 +++--- src/awkward/contents/indexedoptionarray.py | 6 +++--- src/awkward/contents/listarray.py | 8 ++++---- src/awkward/contents/listoffsetarray.py | 8 ++++---- src/awkward/contents/numpyarray.py | 6 +++--- src/awkward/contents/recordarray.py | 6 +++--- src/awkward/contents/regulararray.py | 8 ++++---- src/awkward/contents/unionarray.py | 6 +++--- src/awkward/contents/unmaskedarray.py | 6 +++--- 11 files changed, 44 insertions(+), 38 deletions(-) diff --git a/src/awkward/contents/bytemaskedarray.py b/src/awkward/contents/bytemaskedarray.py index 67eedb7b48..4aaccbbf84 100644 --- a/src/awkward/contents/bytemaskedarray.py +++ b/src/awkward/contents/bytemaskedarray.py @@ -654,14 +654,14 @@ def _fill_none(self, value: Content) -> Content: return self.to_IndexedOptionArray64()._fill_none(value) def _local_index(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: _, nextcarry, outindex = self._nextcarry_outindex(self._backend) next = self._content._carry(nextcarry, False) - out = next._local_index(posaxis, depth) + out = next._local_index(axis, depth) return ak.contents.IndexedOptionArray.simplified( outindex, out, parameters=self._parameters ) diff --git a/src/awkward/contents/emptyarray.py b/src/awkward/contents/emptyarray.py index 45fb779a25..22af894476 100644 --- a/src/awkward/contents/emptyarray.py +++ b/src/awkward/contents/emptyarray.py @@ -189,11 +189,17 @@ def _fill_none(self, value: Content) -> Content: return EmptyArray(parameters=self._parameters, backend=self._backend) def _local_index(self, axis, depth): - return ak.contents.NumpyArray( - self._backend.nplike.empty(0, np.int64), - parameters=None, - backend=self._backend, - ) + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: + return ak.contents.NumpyArray( + self._backend.nplike.empty(0, np.int64), + parameters=None, + backend=self._backend, + ) + else: + raise ak._errors.wrap_error( + np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") + ) def _numbers_to_type(self, name): return ak.contents.EmptyArray( diff --git a/src/awkward/contents/indexedarray.py b/src/awkward/contents/indexedarray.py index 62ea48ceab..1fafc0de74 100644 --- a/src/awkward/contents/indexedarray.py +++ b/src/awkward/contents/indexedarray.py @@ -591,11 +591,11 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: - return self.project()._local_index(posaxis, depth) + return self.project()._local_index(axis, depth) def _unique_index(self, index, sorted=True): next = ak.index.Index64.zeros(self.length, nplike=self._backend.index_nplike) diff --git a/src/awkward/contents/indexedoptionarray.py b/src/awkward/contents/indexedoptionarray.py index 5821961f94..3d5665fc20 100644 --- a/src/awkward/contents/indexedoptionarray.py +++ b/src/awkward/contents/indexedoptionarray.py @@ -740,14 +740,14 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: _, nextcarry, outindex = self._nextcarry_outindex(self._backend) next = self._content._carry(nextcarry, False) - out = next._local_index(posaxis, depth) + out = next._local_index(axis, depth) out2 = ak.contents.IndexedOptionArray( outindex, out, parameters=self._parameters ) diff --git a/src/awkward/contents/listarray.py b/src/awkward/contents/listarray.py index bd023d47bf..aec333235e 100644 --- a/src/awkward/contents/listarray.py +++ b/src/awkward/contents/listarray.py @@ -1068,10 +1068,10 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() - elif posaxis + 1 == depth + 1: + elif posaxis is not None and posaxis + 1 == depth + 1: offsets = self._compact_offsets64(True) if self._backend.nplike.known_data: innerlength = offsets[offsets.length - 1] @@ -1100,7 +1100,7 @@ def _local_index(self, axis, depth): return ak.contents.ListArray( self._starts, self._stops, - self._content._local_index(posaxis, depth + 1), + self._content._local_index(axis, depth + 1), ) def _numbers_to_type(self, name): diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py index 5efbb4a672..160442fe53 100644 --- a/src/awkward/contents/listoffsetarray.py +++ b/src/awkward/contents/listoffsetarray.py @@ -715,10 +715,10 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() - elif posaxis + 1 == depth + 1: + elif posaxis is not None and posaxis + 1 == depth + 1: offsets = self._compact_offsets64(True) if self._backend.nplike.known_data: innerlength = offsets[offsets.length - 1] @@ -745,7 +745,7 @@ def _local_index(self, axis, depth): ) else: return ak.contents.ListOffsetArray( - self._offsets, self._content._local_index(posaxis, depth + 1) + self._offsets, self._content._local_index(axis, depth + 1) ) def _numbers_to_type(self, name): diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index b0342f3482..f9bdb2c3f1 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -434,15 +434,15 @@ def _fill_none(self, value: Content) -> Content: return self def _local_index(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() elif len(self.shape) <= 1: raise ak._errors.wrap_error( np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") ) else: - return self.to_RegularArray()._local_index(posaxis, depth) + return self.to_RegularArray()._local_index(axis, depth) def to_contiguous(self) -> Self: if self.is_contiguous: diff --git a/src/awkward/contents/recordarray.py b/src/awkward/contents/recordarray.py index 27b2b117ae..d784353b23 100644 --- a/src/awkward/contents/recordarray.py +++ b/src/awkward/contents/recordarray.py @@ -658,13 +658,13 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: contents = [] for content in self._contents: - contents.append(content._local_index(posaxis, depth)) + contents.append(content._local_index(axis, depth)) return RecordArray( contents, self._fields, diff --git a/src/awkward/contents/regulararray.py b/src/awkward/contents/regulararray.py index 63b6c44143..8fafdc467c 100644 --- a/src/awkward/contents/regulararray.py +++ b/src/awkward/contents/regulararray.py @@ -687,10 +687,10 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() - elif posaxis + 1 == depth + 1: + elif posaxis is not None and posaxis + 1 == depth + 1: localindex = ak.index.Index64.empty( self._length * self._size, nplike=self._backend.index_nplike ) @@ -706,7 +706,7 @@ def _local_index(self, axis, depth): ) else: return ak.contents.RegularArray( - self._content._local_index(posaxis, depth + 1), self._size, self._length + self._content._local_index(axis, depth + 1), self._size, self._length ) def _numbers_to_type(self, name): diff --git a/src/awkward/contents/unionarray.py b/src/awkward/contents/unionarray.py index 0133c6fe12..4ea4a6541b 100644 --- a/src/awkward/contents/unionarray.py +++ b/src/awkward/contents/unionarray.py @@ -1113,13 +1113,13 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: contents = [] for content in self._contents: - contents.append(content._local_index(posaxis, depth)) + contents.append(content._local_index(axis, depth)) return UnionArray( self._tags, self._index, diff --git a/src/awkward/contents/unmaskedarray.py b/src/awkward/contents/unmaskedarray.py index a98da564c9..7b92e02a35 100644 --- a/src/awkward/contents/unmaskedarray.py +++ b/src/awkward/contents/unmaskedarray.py @@ -289,12 +289,12 @@ def _fill_none(self, value: Content) -> Content: return self._content._fill_none(value) def _local_index(self, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: return UnmaskedArray( - self._content._local_index(posaxis, depth), parameters=self._parameters + self._content._local_index(axis, depth), parameters=self._parameters ) def _numbers_to_type(self, name): From f6af2539c3318eabe0a226485fb7624fad5107bf Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 02:44:26 -0600 Subject: [PATCH 20/26] Shift Content._pad_none to use the standard 'depth' definition. --- src/awkward/_do.py | 2 +- src/awkward/contents/bytemaskedarray.py | 4 ++-- src/awkward/contents/emptyarray.py | 2 +- src/awkward/contents/indexedarray.py | 4 ++-- src/awkward/contents/indexedoptionarray.py | 4 ++-- src/awkward/contents/listarray.py | 4 ++-- src/awkward/contents/listoffsetarray.py | 4 ++-- src/awkward/contents/numpyarray.py | 2 +- src/awkward/contents/recordarray.py | 2 +- src/awkward/contents/regulararray.py | 4 ++-- src/awkward/contents/unionarray.py | 2 +- src/awkward/contents/unmaskedarray.py | 4 ++-- 12 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/awkward/_do.py b/src/awkward/_do.py index aba7fc39de..1e788cce7c 100644 --- a/src/awkward/_do.py +++ b/src/awkward/_do.py @@ -272,7 +272,7 @@ def unique(layout: Content, axis=None): def pad_none( layout: Content, length: Integral, axis: Integral, clip: bool = False ) -> Content: - return layout._pad_none(length, axis, 0, clip) + return layout._pad_none(length, axis, 1, clip) def completely_flatten( diff --git a/src/awkward/contents/bytemaskedarray.py b/src/awkward/contents/bytemaskedarray.py index 4aaccbbf84..6b7a822004 100644 --- a/src/awkward/contents/bytemaskedarray.py +++ b/src/awkward/contents/bytemaskedarray.py @@ -921,9 +921,9 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: mask = ak.index.Index8(self.mask_as_bool(valid_when=False)) index = ak.index.Index64.empty( mask.length, nplike=self._backend.index_nplike diff --git a/src/awkward/contents/emptyarray.py b/src/awkward/contents/emptyarray.py index 22af894476..e617d7731c 100644 --- a/src/awkward/contents/emptyarray.py +++ b/src/awkward/contents/emptyarray.py @@ -280,7 +280,7 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis != depth: + if posaxis + 1 != depth: raise ak._errors.wrap_error( np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") ) diff --git a/src/awkward/contents/indexedarray.py b/src/awkward/contents/indexedarray.py index 1fafc0de74..6d39f77118 100644 --- a/src/awkward/contents/indexedarray.py +++ b/src/awkward/contents/indexedarray.py @@ -917,9 +917,9 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: return self.project()._pad_none(target, posaxis, depth, clip) else: return ak.contents.IndexedArray( diff --git a/src/awkward/contents/indexedoptionarray.py b/src/awkward/contents/indexedoptionarray.py index 3d5665fc20..6f69d79b61 100644 --- a/src/awkward/contents/indexedoptionarray.py +++ b/src/awkward/contents/indexedoptionarray.py @@ -1397,9 +1397,9 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: mask = ak.index.Index8(self.mask_as_bool(valid_when=False)) index = ak.index.Index64.empty(mask.length, self._backend.index_nplike) assert ( diff --git a/src/awkward/contents/listarray.py b/src/awkward/contents/listarray.py index aec333235e..2ccdc21459 100644 --- a/src/awkward/contents/listarray.py +++ b/src/awkward/contents/listarray.py @@ -1234,9 +1234,9 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): if not clip: posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: min_ = ak.index.Index64.empty(1, self._backend.index_nplike) assert ( min_.nplike is self._backend.index_nplike diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py index 160442fe53..2084be4465 100644 --- a/src/awkward/contents/listoffsetarray.py +++ b/src/awkward/contents/listoffsetarray.py @@ -1713,9 +1713,9 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - if posaxis == depth + 1: + if posaxis + 1 == depth + 1: if not clip: tolength = ak.index.Index64.empty(1, self._backend.index_nplike) offsets_ = ak.index.Index64.empty( diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index f9bdb2c3f1..708a15c76e 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -1167,7 +1167,7 @@ def _pad_none(self, target, axis, depth, clip): elif len(self.shape) > 1 or not self.is_contiguous: return self.to_RegularArray()._pad_none(target, axis, depth, clip) posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis != depth: + if posaxis + 1 != depth: raise ak._errors.wrap_error( np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") ) diff --git a/src/awkward/contents/recordarray.py b/src/awkward/contents/recordarray.py index d784353b23..c3c437299f 100644 --- a/src/awkward/contents/recordarray.py +++ b/src/awkward/contents/recordarray.py @@ -806,7 +806,7 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._pad_none_axis0(target, clip) else: contents = [] diff --git a/src/awkward/contents/regulararray.py b/src/awkward/contents/regulararray.py index 8fafdc467c..53e424b30c 100644 --- a/src/awkward/contents/regulararray.py +++ b/src/awkward/contents/regulararray.py @@ -1120,10 +1120,10 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: if not clip: if target < self._size: return self diff --git a/src/awkward/contents/unionarray.py b/src/awkward/contents/unionarray.py index 4ea4a6541b..6dd8a3785a 100644 --- a/src/awkward/contents/unionarray.py +++ b/src/awkward/contents/unionarray.py @@ -1346,7 +1346,7 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._pad_none_axis0(target, clip) else: contents = [] diff --git a/src/awkward/contents/unmaskedarray.py b/src/awkward/contents/unmaskedarray.py index 7b92e02a35..43e8ac61ee 100644 --- a/src/awkward/contents/unmaskedarray.py +++ b/src/awkward/contents/unmaskedarray.py @@ -415,9 +415,9 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: return self._content._pad_none(target, posaxis, depth, clip) else: return ak.contents.UnmaskedArray( From 8943e17dd49953c060f01a01989e280eee26ec35 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 02:49:21 -0600 Subject: [PATCH 21/26] Content._pad_none now uses maybe_posaxis. --- src/awkward/contents/bytemaskedarray.py | 10 +++++----- src/awkward/contents/emptyarray.py | 4 ++-- src/awkward/contents/indexedarray.py | 10 +++++----- src/awkward/contents/indexedoptionarray.py | 10 +++++----- src/awkward/contents/listarray.py | 8 ++++---- src/awkward/contents/listoffsetarray.py | 8 ++++---- src/awkward/contents/numpyarray.py | 6 +++--- src/awkward/contents/recordarray.py | 6 +++--- src/awkward/contents/regulararray.py | 10 +++++----- src/awkward/contents/unionarray.py | 6 +++--- src/awkward/contents/unmaskedarray.py | 10 +++++----- 11 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/awkward/contents/bytemaskedarray.py b/src/awkward/contents/bytemaskedarray.py index 6b7a822004..75ba88dba9 100644 --- a/src/awkward/contents/bytemaskedarray.py +++ b/src/awkward/contents/bytemaskedarray.py @@ -920,10 +920,10 @@ def _nbytes_part(self): return self.mask._nbytes_part() + self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis + 1 == depth + 1: + elif posaxis is not None and posaxis + 1 == depth + 1: mask = ak.index.Index8(self.mask_as_bool(valid_when=False)) index = ak.index.Index64.empty( mask.length, nplike=self._backend.index_nplike @@ -943,14 +943,14 @@ def _pad_none(self, target, axis, depth, clip): self._mask.length, ) ) - next = self.project()._pad_none(target, posaxis, depth, clip) + next = self.project()._pad_none(target, axis, depth, clip) return ak.contents.IndexedOptionArray.simplified( index, next, parameters=self._parameters ) else: return ak.contents.ByteMaskedArray( self._mask, - self._content._pad_none(target, posaxis, depth, clip), + self._content._pad_none(target, axis, depth, clip), self._valid_when, parameters=self._parameters, ) diff --git a/src/awkward/contents/emptyarray.py b/src/awkward/contents/emptyarray.py index e617d7731c..51115788ae 100644 --- a/src/awkward/contents/emptyarray.py +++ b/src/awkward/contents/emptyarray.py @@ -279,8 +279,8 @@ def _nbytes_part(self): return 0 def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 != depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 != depth: raise ak._errors.wrap_error( np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") ) diff --git a/src/awkward/contents/indexedarray.py b/src/awkward/contents/indexedarray.py index 6d39f77118..d589ded12a 100644 --- a/src/awkward/contents/indexedarray.py +++ b/src/awkward/contents/indexedarray.py @@ -916,15 +916,15 @@ def _nbytes_part(self): return self.index._nbytes_part() + self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis + 1 == depth + 1: - return self.project()._pad_none(target, posaxis, depth, clip) + elif posaxis is not None and posaxis + 1 == depth + 1: + return self.project()._pad_none(target, axis, depth, clip) else: return ak.contents.IndexedArray( self._index, - self._content._pad_none(target, posaxis, depth, clip), + self._content._pad_none(target, axis, depth, clip), parameters=self._parameters, ) diff --git a/src/awkward/contents/indexedoptionarray.py b/src/awkward/contents/indexedoptionarray.py index 6f69d79b61..323dc321ff 100644 --- a/src/awkward/contents/indexedoptionarray.py +++ b/src/awkward/contents/indexedoptionarray.py @@ -1396,10 +1396,10 @@ def _nbytes_part(self): return self.index._nbytes_part() + self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis + 1 == depth + 1: + elif posaxis is not None and posaxis + 1 == depth + 1: mask = ak.index.Index8(self.mask_as_bool(valid_when=False)) index = ak.index.Index64.empty(mask.length, self._backend.index_nplike) assert ( @@ -1413,14 +1413,14 @@ def _pad_none(self, target, axis, depth, clip): mask.dtype.type, ](index.data, mask.data, mask.length) ) - next = self.project()._pad_none(target, posaxis, depth, clip) + next = self.project()._pad_none(target, axis, depth, clip) return ak.contents.IndexedOptionArray.simplified( index, next, parameters=self._parameters ) else: return ak.contents.IndexedOptionArray( self._index, - self._content._pad_none(target, posaxis, depth, clip), + self._content._pad_none(target, axis, depth, clip), parameters=self._parameters, ) diff --git a/src/awkward/contents/listarray.py b/src/awkward/contents/listarray.py index 2ccdc21459..3337b3c40c 100644 --- a/src/awkward/contents/listarray.py +++ b/src/awkward/contents/listarray.py @@ -1233,10 +1233,10 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): if not clip: - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis + 1 == depth + 1: + elif posaxis is not None and posaxis + 1 == depth + 1: min_ = ak.index.Index64.empty(1, self._backend.index_nplike) assert ( min_.nplike is self._backend.index_nplike @@ -1329,7 +1329,7 @@ def _pad_none(self, target, axis, depth, clip): return ak.contents.ListArray( self._starts, self._stops, - self._content._pad_none(target, posaxis, depth + 1, clip), + self._content._pad_none(target, axis, depth + 1, clip), parameters=self._parameters, ) else: diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py index 2084be4465..6332882477 100644 --- a/src/awkward/contents/listoffsetarray.py +++ b/src/awkward/contents/listoffsetarray.py @@ -1712,10 +1712,10 @@ def _nbytes_part(self): return self.offsets._nbytes_part() + self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - if posaxis + 1 == depth + 1: + if posaxis is not None and posaxis + 1 == depth + 1: if not clip: tolength = ak.index.Index64.empty(1, self._backend.index_nplike) offsets_ = ak.index.Index64.empty( @@ -1821,7 +1821,7 @@ def _pad_none(self, target, axis, depth, clip): else: return ak.contents.ListOffsetArray( self._offsets, - self._content._pad_none(target, posaxis, depth + 1, clip), + self._content._pad_none(target, axis, depth + 1, clip), parameters=self._parameters, ) diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index 708a15c76e..1588c71406 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -1166,8 +1166,8 @@ def _pad_none(self, target, axis, depth, clip): ) elif len(self.shape) > 1 or not self.is_contiguous: return self.to_RegularArray()._pad_none(target, axis, depth, clip) - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 != depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 != depth: raise ak._errors.wrap_error( np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") ) @@ -1175,7 +1175,7 @@ def _pad_none(self, target, axis, depth, clip): if target < self.length: return self else: - return self._pad_none(target, posaxis, depth, clip=True) + return self._pad_none(target, axis, depth, clip=True) else: return self._pad_none_axis0(target, clip=True) diff --git a/src/awkward/contents/recordarray.py b/src/awkward/contents/recordarray.py index c3c437299f..33995f4b2f 100644 --- a/src/awkward/contents/recordarray.py +++ b/src/awkward/contents/recordarray.py @@ -805,13 +805,13 @@ def _nbytes_part(self): return result def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) else: contents = [] for content in self._contents: - contents.append(content._pad_none(target, posaxis, depth, clip)) + contents.append(content._pad_none(target, axis, depth, clip)) if len(contents) == 0: return ak.contents.RecordArray( contents, diff --git a/src/awkward/contents/regulararray.py b/src/awkward/contents/regulararray.py index 53e424b30c..43079593f8 100644 --- a/src/awkward/contents/regulararray.py +++ b/src/awkward/contents/regulararray.py @@ -1119,16 +1119,16 @@ def _nbytes_part(self): return self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis + 1 == depth + 1: + elif posaxis is not None and posaxis + 1 == depth + 1: if not clip: if target < self._size: return self else: - return self._pad_none(target, posaxis, depth, True) + return self._pad_none(target, axis, depth, True) else: index = ak.index.Index64.empty( @@ -1152,7 +1152,7 @@ def _pad_none(self, target, axis, depth, clip): else: return ak.contents.RegularArray( - self._content._pad_none(target, posaxis, depth + 1, clip), + self._content._pad_none(target, axis, depth + 1, clip), self._size, self._length, parameters=self._parameters, diff --git a/src/awkward/contents/unionarray.py b/src/awkward/contents/unionarray.py index 6dd8a3785a..ddb11fe352 100644 --- a/src/awkward/contents/unionarray.py +++ b/src/awkward/contents/unionarray.py @@ -1345,13 +1345,13 @@ def _nbytes_part(self): return result def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) else: contents = [] for content in self._contents: - contents.append(content._pad_none(target, posaxis, depth, clip)) + contents.append(content._pad_none(target, axis, depth, clip)) return ak.contents.UnionArray.simplified( self.tags, self.index, diff --git a/src/awkward/contents/unmaskedarray.py b/src/awkward/contents/unmaskedarray.py index 43e8ac61ee..06a4bed153 100644 --- a/src/awkward/contents/unmaskedarray.py +++ b/src/awkward/contents/unmaskedarray.py @@ -414,14 +414,14 @@ def _nbytes_part(self): return self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) - elif posaxis + 1 == depth + 1: - return self._content._pad_none(target, posaxis, depth, clip) + elif posaxis is not None and posaxis + 1 == depth + 1: + return self._content._pad_none(target, axis, depth, clip) else: return ak.contents.UnmaskedArray( - self._content._pad_none(target, posaxis, depth, clip), + self._content._pad_none(target, axis, depth, clip), parameters=self._parameters, ) From 82d196de2f082b40740d37d23213acbb14c5f053 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 02:52:51 -0600 Subject: [PATCH 22/26] Shift Content._combinations to use the standard 'depth' definition. --- src/awkward/_do.py | 2 +- src/awkward/contents/bytemaskedarray.py | 2 +- src/awkward/contents/indexedarray.py | 2 +- src/awkward/contents/indexedoptionarray.py | 2 +- src/awkward/contents/listoffsetarray.py | 4 ++-- src/awkward/contents/numpyarray.py | 2 +- src/awkward/contents/recordarray.py | 2 +- src/awkward/contents/regulararray.py | 4 ++-- src/awkward/contents/unionarray.py | 2 +- src/awkward/contents/unmaskedarray.py | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/awkward/_do.py b/src/awkward/_do.py index 1e788cce7c..0f434dc075 100644 --- a/src/awkward/_do.py +++ b/src/awkward/_do.py @@ -208,7 +208,7 @@ def combinations( raise ak._errors.wrap_error( ValueError("if provided, the length of 'fields' must be 'n'") ) - return layout._combinations(n, replacement, recordlookup, parameters, axis, 0) + return layout._combinations(n, replacement, recordlookup, parameters, axis, 1) def is_unique(layout, axis: Integral | None = None) -> bool: diff --git a/src/awkward/contents/bytemaskedarray.py b/src/awkward/contents/bytemaskedarray.py index 75ba88dba9..dac1d3a856 100644 --- a/src/awkward/contents/bytemaskedarray.py +++ b/src/awkward/contents/bytemaskedarray.py @@ -732,7 +732,7 @@ def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): ValueError("in combinations, 'n' must be at least 1") ) posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: _, nextcarry, outindex = self._nextcarry_outindex(self._backend) diff --git a/src/awkward/contents/indexedarray.py b/src/awkward/contents/indexedarray.py index d589ded12a..3aece3f38c 100644 --- a/src/awkward/contents/indexedarray.py +++ b/src/awkward/contents/indexedarray.py @@ -861,7 +861,7 @@ def _sort_next( def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: return self.project()._combinations( diff --git a/src/awkward/contents/indexedoptionarray.py b/src/awkward/contents/indexedoptionarray.py index 323dc321ff..3a5ec9beed 100644 --- a/src/awkward/contents/indexedoptionarray.py +++ b/src/awkward/contents/indexedoptionarray.py @@ -1362,7 +1362,7 @@ def _reduce_next( def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: _, nextcarry, outindex = self._nextcarry_outindex(self._backend) diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py index 6332882477..79938ed950 100644 --- a/src/awkward/contents/listoffsetarray.py +++ b/src/awkward/contents/listoffsetarray.py @@ -1270,9 +1270,9 @@ def _sort_next( def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: if ( self.parameter("__array__") == "string" or self.parameter("__array__") == "bytestring" diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index 1588c71406..086fb537e5 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -1021,7 +1021,7 @@ def _sort_next( def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) elif len(self.shape) <= 1: raise ak._errors.wrap_error( diff --git a/src/awkward/contents/recordarray.py b/src/awkward/contents/recordarray.py index 33995f4b2f..67b304c327 100644 --- a/src/awkward/contents/recordarray.py +++ b/src/awkward/contents/recordarray.py @@ -742,7 +742,7 @@ def _sort_next( def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: contents = [] diff --git a/src/awkward/contents/regulararray.py b/src/awkward/contents/regulararray.py index 43079593f8..6a4357bf77 100644 --- a/src/awkward/contents/regulararray.py +++ b/src/awkward/contents/regulararray.py @@ -815,9 +815,9 @@ def _sort_next( def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) - elif posaxis == depth + 1: + elif posaxis + 1 == depth + 1: if ( self.parameter("__array__") == "string" or self.parameter("__array__") == "bytestring" diff --git a/src/awkward/contents/unionarray.py b/src/awkward/contents/unionarray.py index ddb11fe352..e2712a1cb2 100644 --- a/src/awkward/contents/unionarray.py +++ b/src/awkward/contents/unionarray.py @@ -1129,7 +1129,7 @@ def _local_index(self, axis, depth): def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: contents = [] diff --git a/src/awkward/contents/unmaskedarray.py b/src/awkward/contents/unmaskedarray.py index 06a4bed153..14f4ea93fc 100644 --- a/src/awkward/contents/unmaskedarray.py +++ b/src/awkward/contents/unmaskedarray.py @@ -373,7 +373,7 @@ def _sort_next( def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis == depth: + if posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: return ak.contents.UnmaskedArray( From 3b84fecedd0a336608a98438451f0ff94e9f3993 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 02:57:09 -0600 Subject: [PATCH 23/26] Content._combinations now uses maybe_posaxis. --- src/awkward/contents/bytemaskedarray.py | 6 +++--- src/awkward/contents/indexedarray.py | 6 +++--- src/awkward/contents/indexedoptionarray.py | 6 +++--- src/awkward/contents/listoffsetarray.py | 8 ++++---- src/awkward/contents/numpyarray.py | 6 +++--- src/awkward/contents/recordarray.py | 6 +++--- src/awkward/contents/regulararray.py | 10 ++++------ src/awkward/contents/unionarray.py | 6 +++--- src/awkward/contents/unmaskedarray.py | 6 +++--- 9 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/awkward/contents/bytemaskedarray.py b/src/awkward/contents/bytemaskedarray.py index dac1d3a856..086aaecc69 100644 --- a/src/awkward/contents/bytemaskedarray.py +++ b/src/awkward/contents/bytemaskedarray.py @@ -731,15 +731,15 @@ def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): raise ak._errors.wrap_error( ValueError("in combinations, 'n' must be at least 1") ) - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: _, nextcarry, outindex = self._nextcarry_outindex(self._backend) next = self._content._carry(nextcarry, True) out = next._combinations( - n, replacement, recordlookup, parameters, posaxis, depth + n, replacement, recordlookup, parameters, axis, depth ) return ak.contents.IndexedOptionArray.simplified( outindex, out, parameters=parameters diff --git a/src/awkward/contents/indexedarray.py b/src/awkward/contents/indexedarray.py index 3aece3f38c..f2c80ce435 100644 --- a/src/awkward/contents/indexedarray.py +++ b/src/awkward/contents/indexedarray.py @@ -860,12 +860,12 @@ def _sort_next( ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis) + if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: return self.project()._combinations( - n, replacement, recordlookup, parameters, posaxis, depth + n, replacement, recordlookup, parameters, axis, depth ) def _reduce_next( diff --git a/src/awkward/contents/indexedoptionarray.py b/src/awkward/contents/indexedoptionarray.py index 3a5ec9beed..7230cefc07 100644 --- a/src/awkward/contents/indexedoptionarray.py +++ b/src/awkward/contents/indexedoptionarray.py @@ -1361,14 +1361,14 @@ def _reduce_next( ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: _, nextcarry, outindex = self._nextcarry_outindex(self._backend) next = self._content._carry(nextcarry, True) out = next._combinations( - n, replacement, recordlookup, parameters, posaxis, depth + n, replacement, recordlookup, parameters, axis, depth ) return IndexedOptionArray.simplified(outindex, out, parameters=parameters) diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py index 79938ed950..db6f0f637e 100644 --- a/src/awkward/contents/listoffsetarray.py +++ b/src/awkward/contents/listoffsetarray.py @@ -1269,10 +1269,10 @@ def _sort_next( ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) - elif posaxis + 1 == depth + 1: + elif posaxis is not None and posaxis + 1 == depth + 1: if ( self.parameter("__array__") == "string" or self.parameter("__array__") == "bytestring" @@ -1381,7 +1381,7 @@ def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): else: compact = self.to_ListOffsetArray64(True) next = compact._content._combinations( - n, replacement, recordlookup, parameters, posaxis, depth + 1 + n, replacement, recordlookup, parameters, axis, depth + 1 ) return ak.contents.ListOffsetArray( compact.offsets, next, parameters=self._parameters diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index 086fb537e5..e6b3557af8 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -1020,8 +1020,8 @@ def _sort_next( ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) elif len(self.shape) <= 1: raise ak._errors.wrap_error( @@ -1029,7 +1029,7 @@ def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): ) else: return self.to_RegularArray()._combinations( - n, replacement, recordlookup, parameters, posaxis, depth + n, replacement, recordlookup, parameters, axis, depth ) def _reduce_next( diff --git a/src/awkward/contents/recordarray.py b/src/awkward/contents/recordarray.py index 67b304c327..283acf161a 100644 --- a/src/awkward/contents/recordarray.py +++ b/src/awkward/contents/recordarray.py @@ -741,15 +741,15 @@ def _sort_next( ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: contents = [] for content in self._contents: contents.append( content._combinations( - n, replacement, recordlookup, parameters, posaxis, depth + n, replacement, recordlookup, parameters, axis, depth ) ) return ak.contents.RecordArray( diff --git a/src/awkward/contents/regulararray.py b/src/awkward/contents/regulararray.py index 6a4357bf77..5829748a0e 100644 --- a/src/awkward/contents/regulararray.py +++ b/src/awkward/contents/regulararray.py @@ -814,10 +814,10 @@ def _sort_next( return out def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) - elif posaxis + 1 == depth + 1: + elif posaxis is not None and posaxis + 1 == depth + 1: if ( self.parameter("__array__") == "string" or self.parameter("__array__") == "bytestring" @@ -905,9 +905,7 @@ def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): else: next = self._content._getitem_range( slice(0, self._length * self._size) - )._combinations( - n, replacement, recordlookup, parameters, posaxis, depth + 1 - ) + )._combinations(n, replacement, recordlookup, parameters, axis, depth + 1) return ak.contents.RegularArray( next, self._size, self._length, parameters=self._parameters ) diff --git a/src/awkward/contents/unionarray.py b/src/awkward/contents/unionarray.py index e2712a1cb2..888182f65c 100644 --- a/src/awkward/contents/unionarray.py +++ b/src/awkward/contents/unionarray.py @@ -1128,15 +1128,15 @@ def _local_index(self, axis, depth): ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: contents = [] for content in self._contents: contents.append( content._combinations( - n, replacement, recordlookup, parameters, posaxis, depth + n, replacement, recordlookup, parameters, axis, depth ) ) return ak.unionarray.UnionArray( diff --git a/src/awkward/contents/unmaskedarray.py b/src/awkward/contents/unmaskedarray.py index 14f4ea93fc..4a7abfbbae 100644 --- a/src/awkward/contents/unmaskedarray.py +++ b/src/awkward/contents/unmaskedarray.py @@ -372,13 +372,13 @@ def _sort_next( return out def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.axis_wrap_if_negative(self, axis) - if posaxis + 1 == depth: + posaxis = ak._do.maybe_posaxis(self, axis, depth) + if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: return ak.contents.UnmaskedArray( self._content._combinations( - n, replacement, recordlookup, parameters, posaxis, depth + n, replacement, recordlookup, parameters, axis, depth ), parameters=self._parameters, ) From 08f4c407ee388776020063cde936b11c8ae0d76f Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 03:03:06 -0600 Subject: [PATCH 24/26] Delete axis_wrap_if_negative and move maybe_posaxis to _util. --- src/awkward/_do.py | 64 +--------------------- src/awkward/_util.py | 19 +++++++ src/awkward/contents/bytemaskedarray.py | 8 +-- src/awkward/contents/emptyarray.py | 6 +- src/awkward/contents/indexedarray.py | 8 +-- src/awkward/contents/indexedoptionarray.py | 8 +-- src/awkward/contents/listarray.py | 4 +- src/awkward/contents/listoffsetarray.py | 8 +-- src/awkward/contents/numpyarray.py | 8 +-- src/awkward/contents/recordarray.py | 8 +-- src/awkward/contents/regulararray.py | 6 +- src/awkward/contents/unionarray.py | 8 +-- src/awkward/contents/unmaskedarray.py | 8 +-- src/awkward/operations/ak_cartesian.py | 4 +- src/awkward/operations/ak_concatenate.py | 6 +- src/awkward/operations/ak_fill_none.py | 2 +- src/awkward/operations/ak_firsts.py | 4 +- src/awkward/operations/ak_flatten.py | 2 +- src/awkward/operations/ak_from_regular.py | 4 +- src/awkward/operations/ak_is_none.py | 2 +- src/awkward/operations/ak_num.py | 4 +- src/awkward/operations/ak_ptp.py | 2 +- src/awkward/operations/ak_singletons.py | 2 +- src/awkward/operations/ak_to_regular.py | 4 +- src/awkward/operations/ak_unflatten.py | 4 +- 25 files changed, 80 insertions(+), 123 deletions(-) diff --git a/src/awkward/_do.py b/src/awkward/_do.py index 0f434dc075..ec314b2dc6 100644 --- a/src/awkward/_do.py +++ b/src/awkward/_do.py @@ -7,7 +7,7 @@ import awkward as ak from awkward._backends import Backend -from awkward.contents.content import ActionType, AxisMaybeNone, Content +from awkward.contents.content import ActionType, Content from awkward.forms import form from awkward.record import Record from awkward.typing import Any @@ -122,68 +122,6 @@ def getkey(layout, form, attribute): return form, len(content), container -# FIXME: eventually, this should be in ak._util, but it's here to juxtapose axis_wrap_if_negative -# This function must depend on 'depth'. I don't know how axis_wrap_if_negative was believed to work. -def maybe_posaxis( - layout: Content | Record, axis: AxisMaybeNone, depth: Integral -) -> AxisMaybeNone: - if isinstance(layout, Record): - if axis == 0: - raise ak._errors.wrap_error( - np.AxisError("Record type at axis=0 is a scalar, not an array") - ) - return maybe_posaxis(layout._array, axis, depth) - - if axis >= 0: - return axis - - else: - is_branching, additional_depth = layout.branch_depth - if not is_branching: - return axis + depth + additional_depth - 1 - else: - return None - - -# FIXME: eventually phase out the use of axis_wrap_if_negative in favor of maybe_posaxis -def axis_wrap_if_negative( - layout: Content | Record, axis: AxisMaybeNone -) -> AxisMaybeNone: - if isinstance(layout, Record): - if axis == 0: - raise ak._errors.wrap_error( - np.AxisError("Record type at axis=0 is a scalar, not an array") - ) - return axis_wrap_if_negative(layout._array, axis) - - else: - if axis is None or axis >= 0: - return axis - - mindepth, maxdepth = layout.minmax_depth - depth = layout.purelist_depth - if mindepth == depth and maxdepth == depth: - posaxis = depth + axis - if posaxis < 0: - raise ak._errors.wrap_error( - np.AxisError( - f"axis={axis} exceeds the depth ({depth}) of this array" - ) - ) - return posaxis - - elif mindepth + axis == 0: - raise ak._errors.wrap_error( - np.AxisError( - "axis={} exceeds the depth ({}) of at least one record field (or union possibility) of this array".format( - axis, depth - ) - ) - ) - - return axis - - def local_index(layout: Content, axis: Integral): return layout._local_index(axis, 1) diff --git a/src/awkward/_util.py b/src/awkward/_util.py index 9de873c4d8..b9b2fdb370 100644 --- a/src/awkward/_util.py +++ b/src/awkward/_util.py @@ -825,3 +825,22 @@ def _impl(array): raise ak._errors.wrap_error( ValueError(f"{module.__name__} is not supported by to_arraylib") ) + + +def maybe_posaxis(layout, axis, depth): + if isinstance(layout, ak.record.Record): + if axis == 0: + raise ak._errors.wrap_error( + np.AxisError("Record type at axis=0 is a scalar, not an array") + ) + return maybe_posaxis(layout._array, axis, depth) + + if axis >= 0: + return axis + + else: + is_branching, additional_depth = layout.branch_depth + if not is_branching: + return axis + depth + additional_depth - 1 + else: + return None diff --git a/src/awkward/contents/bytemaskedarray.py b/src/awkward/contents/bytemaskedarray.py index 086aaecc69..570f4f78b5 100644 --- a/src/awkward/contents/bytemaskedarray.py +++ b/src/awkward/contents/bytemaskedarray.py @@ -557,7 +557,7 @@ def project(self, mask=None): return self._content._carry(nextcarry, False) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: @@ -654,7 +654,7 @@ def _fill_none(self, value: Content) -> Content: return self.to_IndexedOptionArray64()._fill_none(value) def _local_index(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: @@ -731,7 +731,7 @@ def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): raise ak._errors.wrap_error( ValueError("in combinations, 'n' must be at least 1") ) - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: @@ -920,7 +920,7 @@ def _nbytes_part(self): return self.mask._nbytes_part() + self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) elif posaxis is not None and posaxis + 1 == depth + 1: diff --git a/src/awkward/contents/emptyarray.py b/src/awkward/contents/emptyarray.py index 51115788ae..4362946930 100644 --- a/src/awkward/contents/emptyarray.py +++ b/src/awkward/contents/emptyarray.py @@ -159,7 +159,7 @@ def _getitem_next(self, head, tail, advanced): raise ak._errors.wrap_error(AssertionError(repr(head))) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error( np.AxisError(self, "axis=0 not allowed for flatten") @@ -189,7 +189,7 @@ def _fill_none(self, value: Content) -> Content: return EmptyArray(parameters=self._parameters, backend=self._backend) def _local_index(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return ak.contents.NumpyArray( self._backend.nplike.empty(0, np.int64), @@ -279,7 +279,7 @@ def _nbytes_part(self): return 0 def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 != depth: raise ak._errors.wrap_error( np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") diff --git a/src/awkward/contents/indexedarray.py b/src/awkward/contents/indexedarray.py index f2c80ce435..c5a3c76973 100644 --- a/src/awkward/contents/indexedarray.py +++ b/src/awkward/contents/indexedarray.py @@ -389,7 +389,7 @@ def project(self, mask=None): ) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) @@ -591,7 +591,7 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: @@ -860,7 +860,7 @@ def _sort_next( ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis) + posaxis = ak._util.maybe_posaxis(self, axis) if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: @@ -916,7 +916,7 @@ def _nbytes_part(self): return self.index._nbytes_part() + self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) elif posaxis is not None and posaxis + 1 == depth + 1: diff --git a/src/awkward/contents/indexedoptionarray.py b/src/awkward/contents/indexedoptionarray.py index 7230cefc07..97a1977c83 100644 --- a/src/awkward/contents/indexedoptionarray.py +++ b/src/awkward/contents/indexedoptionarray.py @@ -476,7 +476,7 @@ def project(self, mask=None): return self._content._carry(nextcarry, False) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: @@ -740,7 +740,7 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: @@ -1361,7 +1361,7 @@ def _reduce_next( ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: @@ -1396,7 +1396,7 @@ def _nbytes_part(self): return self.index._nbytes_part() + self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) elif posaxis is not None and posaxis + 1 == depth + 1: diff --git a/src/awkward/contents/listarray.py b/src/awkward/contents/listarray.py index 3337b3c40c..efd60e9cb9 100644 --- a/src/awkward/contents/listarray.py +++ b/src/awkward/contents/listarray.py @@ -1068,7 +1068,7 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() elif posaxis is not None and posaxis + 1 == depth + 1: @@ -1233,7 +1233,7 @@ def _nbytes_part(self): def _pad_none(self, target, axis, depth, clip): if not clip: - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) elif posaxis is not None and posaxis + 1 == depth + 1: diff --git a/src/awkward/contents/listoffsetarray.py b/src/awkward/contents/listoffsetarray.py index db6f0f637e..496f855c01 100644 --- a/src/awkward/contents/listoffsetarray.py +++ b/src/awkward/contents/listoffsetarray.py @@ -602,7 +602,7 @@ def _getitem_next(self, head, tail, advanced): raise ak._errors.wrap_error(AssertionError(repr(head))) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) @@ -715,7 +715,7 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() elif posaxis is not None and posaxis + 1 == depth + 1: @@ -1269,7 +1269,7 @@ def _sort_next( ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) elif posaxis is not None and posaxis + 1 == depth + 1: @@ -1712,7 +1712,7 @@ def _nbytes_part(self): return self.offsets._nbytes_part() + self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) if posaxis is not None and posaxis + 1 == depth + 1: diff --git a/src/awkward/contents/numpyarray.py b/src/awkward/contents/numpyarray.py index e6b3557af8..a05d5489e2 100644 --- a/src/awkward/contents/numpyarray.py +++ b/src/awkward/contents/numpyarray.py @@ -316,7 +316,7 @@ def _getitem_next(self, head, tail, advanced): raise ak._errors.wrap_error(AssertionError(repr(head))) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) @@ -434,7 +434,7 @@ def _fill_none(self, value: Content) -> Content: return self def _local_index(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() elif len(self.shape) <= 1: @@ -1020,7 +1020,7 @@ def _sort_next( ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) elif len(self.shape) <= 1: @@ -1166,7 +1166,7 @@ def _pad_none(self, target, axis, depth, clip): ) elif len(self.shape) > 1 or not self.is_contiguous: return self.to_RegularArray()._pad_none(target, axis, depth, clip) - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 != depth: raise ak._errors.wrap_error( np.AxisError(f"axis={axis} exceeds the depth of this array ({depth})") diff --git a/src/awkward/contents/recordarray.py b/src/awkward/contents/recordarray.py index 283acf161a..b2f37d756b 100644 --- a/src/awkward/contents/recordarray.py +++ b/src/awkward/contents/recordarray.py @@ -444,7 +444,7 @@ def _getitem_next(self, head, tail, advanced): return next._getitem_next(nexthead, nexttail, advanced) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) @@ -658,7 +658,7 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: @@ -741,7 +741,7 @@ def _sort_next( ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: @@ -805,7 +805,7 @@ def _nbytes_part(self): return result def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) else: diff --git a/src/awkward/contents/regulararray.py b/src/awkward/contents/regulararray.py index 5829748a0e..aa313383c0 100644 --- a/src/awkward/contents/regulararray.py +++ b/src/awkward/contents/regulararray.py @@ -687,7 +687,7 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() elif posaxis is not None and posaxis + 1 == depth + 1: @@ -814,7 +814,7 @@ def _sort_next( return out def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) elif posaxis is not None and posaxis + 1 == depth + 1: @@ -1117,7 +1117,7 @@ def _nbytes_part(self): return self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) diff --git a/src/awkward/contents/unionarray.py b/src/awkward/contents/unionarray.py index 888182f65c..1968d29ae1 100644 --- a/src/awkward/contents/unionarray.py +++ b/src/awkward/contents/unionarray.py @@ -760,7 +760,7 @@ def _getitem_next(self, head, tail, advanced): raise ak._errors.wrap_error(AssertionError(repr(head))) def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) @@ -1113,7 +1113,7 @@ def _fill_none(self, value: Content) -> Content: ) def _local_index(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: @@ -1128,7 +1128,7 @@ def _local_index(self, axis, depth): ) def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: @@ -1345,7 +1345,7 @@ def _nbytes_part(self): return result def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) else: diff --git a/src/awkward/contents/unmaskedarray.py b/src/awkward/contents/unmaskedarray.py index 4a7abfbbae..b0c405952f 100644 --- a/src/awkward/contents/unmaskedarray.py +++ b/src/awkward/contents/unmaskedarray.py @@ -234,7 +234,7 @@ def project(self, mask=None): return self._content def _offsets_and_flattened(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: raise ak._errors.wrap_error(np.AxisError("axis=0 not allowed for flatten")) else: @@ -289,7 +289,7 @@ def _fill_none(self, value: Content) -> Content: return self._content._fill_none(value) def _local_index(self, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._local_index_axis0() else: @@ -372,7 +372,7 @@ def _sort_next( return out def _combinations(self, n, replacement, recordlookup, parameters, axis, depth): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._combinations_axis0(n, replacement, recordlookup, parameters) else: @@ -414,7 +414,7 @@ def _nbytes_part(self): return self.content._nbytes_part() def _pad_none(self, target, axis, depth, clip): - posaxis = ak._do.maybe_posaxis(self, axis, depth) + posaxis = ak._util.maybe_posaxis(self, axis, depth) if posaxis is not None and posaxis + 1 == depth: return self._pad_none_axis0(target, clip) elif posaxis is not None and posaxis + 1 == depth + 1: diff --git a/src/awkward/operations/ak_cartesian.py b/src/awkward/operations/ak_cartesian.py index d95b86e706..c1cc23a377 100644 --- a/src/awkward/operations/ak_cartesian.py +++ b/src/awkward/operations/ak_cartesian.py @@ -235,11 +235,11 @@ def _impl(arrays, axis, nested, parameters, with_name, highlevel, behavior): else: new_arrays_values = new_arrays - posaxis = ak._do.maybe_posaxis(new_arrays_values[0], axis, 1) + posaxis = ak._util.maybe_posaxis(new_arrays_values[0], axis, 1) if posaxis is None or posaxis < 0: raise ak._errors.wrap_error(ValueError("negative axis depth is ambiguous")) for x in new_arrays_values[1:]: - if ak._do.maybe_posaxis(x, axis, 1) != posaxis: + if ak._util.maybe_posaxis(x, axis, 1) != posaxis: raise ak._errors.wrap_error( ValueError( "arrays to cartesian-product do not have the same depth for negative axis" diff --git a/src/awkward/operations/ak_concatenate.py b/src/awkward/operations/ak_concatenate.py index 915d331f29..899c096c54 100644 --- a/src/awkward/operations/ak_concatenate.py +++ b/src/awkward/operations/ak_concatenate.py @@ -56,7 +56,7 @@ def _impl(arrays, axis, mergebool, highlevel, behavior): content = ak.operations.to_layout(arrays, allow_record=False, allow_other=False) # Only handle concatenation along `axis=0` # Let ambiguous depth arrays fall through - if ak._do.maybe_posaxis(content, axis, 1) == 0: + if ak._util.maybe_posaxis(content, axis, 1) == 0: return ak.operations.ak_flatten._impl(content, 1, highlevel, behavior) content_or_others = [ @@ -72,7 +72,7 @@ def _impl(arrays, axis, mergebool, highlevel, behavior): ValueError("need at least one array to concatenate") ) - posaxis = ak._do.maybe_posaxis(contents[0], axis, 1) + posaxis = ak._util.maybe_posaxis(contents[0], axis, 1) maxdepth = max( x.minmax_depth[1] for x in content_or_others @@ -87,7 +87,7 @@ def _impl(arrays, axis, mergebool, highlevel, behavior): ) for x in content_or_others: if isinstance(x, ak.contents.Content): - if ak._do.maybe_posaxis(x, axis, 1) != posaxis: + if ak._util.maybe_posaxis(x, axis, 1) != posaxis: raise ak._errors.wrap_error( ValueError( "arrays to concatenate do not have the same depth for negative " diff --git a/src/awkward/operations/ak_fill_none.py b/src/awkward/operations/ak_fill_none.py index f8b617e0bc..a419f3f01b 100644 --- a/src/awkward/operations/ak_fill_none.py +++ b/src/awkward/operations/ak_fill_none.py @@ -120,7 +120,7 @@ def action(layout, continuation, **kwargs): else: def action(layout, depth, **kwargs): - posaxis = ak._do.maybe_posaxis(layout, axis, depth) + posaxis = ak._util.maybe_posaxis(layout, axis, depth) if posaxis is not None and posaxis + 1 < depth: return layout elif posaxis is not None and posaxis + 1 == depth: diff --git a/src/awkward/operations/ak_firsts.py b/src/awkward/operations/ak_firsts.py index f7c9d7a6b5..d34e67a66f 100644 --- a/src/awkward/operations/ak_firsts.py +++ b/src/awkward/operations/ak_firsts.py @@ -52,7 +52,7 @@ def _impl(array, axis, highlevel, behavior): TypeError(f"'axis' must be an integer, not {axis!r}") ) - if ak._do.maybe_posaxis(layout, axis, 1) == 0: + if ak._util.maybe_posaxis(layout, axis, 1) == 0: # specialized logic; it's tested in test_0582-propagate-context-in-broadcast_and_apply.py # Build an integer-typed slice array, so that we can # ensure we have advanced indexing for both length==0 @@ -66,7 +66,7 @@ def _impl(array, axis, highlevel, behavior): else: def action(layout, depth, depth_context, **kwargs): - posaxis = ak._do.maybe_posaxis(layout, axis, depth) + posaxis = ak._util.maybe_posaxis(layout, axis, depth) if posaxis == depth and layout.is_list: nplike = layout._backend.index_nplike diff --git a/src/awkward/operations/ak_flatten.py b/src/awkward/operations/ak_flatten.py index 85a17440fe..2d4d5d6cf2 100644 --- a/src/awkward/operations/ak_flatten.py +++ b/src/awkward/operations/ak_flatten.py @@ -174,7 +174,7 @@ def _impl(array, axis, highlevel, behavior): return ak._util.wrap(result, behavior, highlevel) - elif axis == 0 or ak._do.maybe_posaxis(layout, axis, 1) == 0: + elif axis == 0 or ak._util.maybe_posaxis(layout, axis, 1) == 0: def apply(layout): backend = layout.backend diff --git a/src/awkward/operations/ak_from_regular.py b/src/awkward/operations/ak_from_regular.py index 124a5f11dd..4705bf4793 100644 --- a/src/awkward/operations/ak_from_regular.py +++ b/src/awkward/operations/ak_from_regular.py @@ -52,13 +52,13 @@ def action(layout, continuation, **kwargs): out = ak._do.recursively_apply(layout, action, behavior, numpy_to_regular=True) - elif ak._do.maybe_posaxis(layout, axis, 1) == 0: + elif ak._util.maybe_posaxis(layout, axis, 1) == 0: out = layout # the top-level is already regular (ArrayType) else: def action(layout, depth, **kwargs): - posaxis = ak._do.maybe_posaxis(layout, axis, depth) + posaxis = ak._util.maybe_posaxis(layout, axis, depth) if posaxis == depth and layout.is_regular: return layout.to_ListOffsetArray64(False) elif posaxis == depth and layout.is_list: diff --git a/src/awkward/operations/ak_is_none.py b/src/awkward/operations/ak_is_none.py index 5f28459b34..cc2806bf87 100644 --- a/src/awkward/operations/ak_is_none.py +++ b/src/awkward/operations/ak_is_none.py @@ -38,7 +38,7 @@ def _impl(array, axis, highlevel, behavior): ) def action(layout, depth, **kwargs): - posaxis = ak._do.maybe_posaxis(layout, axis, depth) + posaxis = ak._util.maybe_posaxis(layout, axis, depth) if posaxis is not None and posaxis + 1 == depth: if layout.is_union: diff --git a/src/awkward/operations/ak_num.py b/src/awkward/operations/ak_num.py index 5428628ac3..84745e6caa 100644 --- a/src/awkward/operations/ak_num.py +++ b/src/awkward/operations/ak_num.py @@ -80,14 +80,14 @@ def _impl(array, axis, highlevel, behavior): TypeError(f"'axis' must be an integer, not {axis!r}") ) - if ak._do.maybe_posaxis(layout, axis, 1) == 0: + if ak._util.maybe_posaxis(layout, axis, 1) == 0: if isinstance(layout, ak.record.Record): return 1 else: return layout.length def action(layout, depth, **kwargs): - posaxis = ak._do.maybe_posaxis(layout, axis, depth) + posaxis = ak._util.maybe_posaxis(layout, axis, depth) if posaxis == depth and layout.is_list: return ak.contents.NumpyArray(layout.stops.data - layout.starts.data) diff --git a/src/awkward/operations/ak_ptp.py b/src/awkward/operations/ak_ptp.py index 77531d0ea4..8b140fd2c7 100644 --- a/src/awkward/operations/ak_ptp.py +++ b/src/awkward/operations/ak_ptp.py @@ -131,7 +131,7 @@ def _impl(array, axis, keepdims, mask_identity, flatten_records): out = ak.highlevel.Array(ak.operations.fill_none(out, 0, axis=-1)) if not keepdims: - posaxis = ak._do.maybe_posaxis(out.layout, axis, 1) + posaxis = ak._util.maybe_posaxis(out.layout, axis, 1) out = out[(slice(None, None),) * posaxis + (0,)] return out diff --git a/src/awkward/operations/ak_singletons.py b/src/awkward/operations/ak_singletons.py index 69ea4aab41..d777be16cd 100644 --- a/src/awkward/operations/ak_singletons.py +++ b/src/awkward/operations/ak_singletons.py @@ -53,7 +53,7 @@ def _impl(array, axis, highlevel, behavior): ) def action(layout, depth, **kwargs): - posaxis = ak._do.maybe_posaxis(layout, axis, depth) + posaxis = ak._util.maybe_posaxis(layout, axis, depth) if posaxis is not None and posaxis + 1 == depth: if layout.is_union: diff --git a/src/awkward/operations/ak_to_regular.py b/src/awkward/operations/ak_to_regular.py index 7ac65a8880..ebbe286776 100644 --- a/src/awkward/operations/ak_to_regular.py +++ b/src/awkward/operations/ak_to_regular.py @@ -65,13 +65,13 @@ def action(layout, continuation, **kwargs): out = ak._do.recursively_apply(layout, action, behavior) - elif ak._do.maybe_posaxis(layout, axis, 1) == 0: + elif ak._util.maybe_posaxis(layout, axis, 1) == 0: out = layout # the top-level can only be regular (ArrayType) else: def action(layout, depth, **kwargs): - posaxis = ak._do.maybe_posaxis(layout, axis, depth) + posaxis = ak._util.maybe_posaxis(layout, axis, depth) if posaxis == depth and layout.is_list: return layout.to_RegularArray() diff --git a/src/awkward/operations/ak_unflatten.py b/src/awkward/operations/ak_unflatten.py index b9c0894c4f..f150197b8d 100644 --- a/src/awkward/operations/ak_unflatten.py +++ b/src/awkward/operations/ak_unflatten.py @@ -156,7 +156,7 @@ def doit(layout): return out - if axis == 0 or ak._do.maybe_posaxis(layout, axis, 1) == 0: + if axis == 0 or ak._util.maybe_posaxis(layout, axis, 1) == 0: out = doit(layout) else: @@ -167,7 +167,7 @@ def transform(layout, depth, axis): # internal layout to be unflattened (#910) layout = layout.to_packed() - posaxis = ak._do.maybe_posaxis(layout, axis, depth) + posaxis = ak._util.maybe_posaxis(layout, axis, depth) if posaxis == depth and layout.is_list: # We are one *above* the level where we want to apply this. listoffsetarray = layout.to_ListOffsetArray64(True) From 18d7a7dd5600faad59ecda2e5eca08cf36c0b9b8 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 9 Dec 2022 09:05:51 -0600 Subject: [PATCH 25/26] Reenterant calling recursively_apply resets depth; this idiom is simpler and correct for deep unions. --- src/awkward/operations/ak_is_none.py | 14 +++----------- src/awkward/operations/ak_singletons.py | 12 ++---------- tests/test_1914-improved-axis-to-posaxis.py | 11 +++++++++++ 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/awkward/operations/ak_is_none.py b/src/awkward/operations/ak_is_none.py index cc2806bf87..b490e1ec07 100644 --- a/src/awkward/operations/ak_is_none.py +++ b/src/awkward/operations/ak_is_none.py @@ -37,20 +37,12 @@ def _impl(array, axis, highlevel, behavior): TypeError(f"'axis' must be an integer, not {axis!r}") ) - def action(layout, depth, **kwargs): + def action(layout, depth, lateral_context, **kwargs): posaxis = ak._util.maybe_posaxis(layout, axis, depth) if posaxis is not None and posaxis + 1 == depth: - if layout.is_union: - contents = [ - ak._do.recursively_apply(x, action, behavior, numpy_to_regular=True) - for x in layout.contents - ] - return ak.contents.UnionArray.simplified( - layout.tags, - layout.index, - contents, - ) + if layout.is_union or layout.is_record: + return None elif layout.is_option: return ak.contents.NumpyArray(layout.mask_as_bool(valid_when=False)) diff --git a/src/awkward/operations/ak_singletons.py b/src/awkward/operations/ak_singletons.py index d777be16cd..2d223ef276 100644 --- a/src/awkward/operations/ak_singletons.py +++ b/src/awkward/operations/ak_singletons.py @@ -56,16 +56,8 @@ def action(layout, depth, **kwargs): posaxis = ak._util.maybe_posaxis(layout, axis, depth) if posaxis is not None and posaxis + 1 == depth: - if layout.is_union: - contents = [ - ak._do.recursively_apply(x, action, behavior, numpy_to_regular=True) - for x in layout.contents - ] - return ak.contents.UnionArray.simplified( - layout.tags, - layout.index, - contents, - ) + if layout.is_union or layout.is_record: + return None elif layout.is_option: nplike = layout._backend.index_nplike diff --git a/tests/test_1914-improved-axis-to-posaxis.py b/tests/test_1914-improved-axis-to-posaxis.py index 38e73046e0..92f5db04e3 100644 --- a/tests/test_1914-improved-axis-to-posaxis.py +++ b/tests/test_1914-improved-axis-to-posaxis.py @@ -129,3 +129,14 @@ def test_singletons(): with pytest.raises(np.AxisError): ak.singletons(array, axis=-3) + + +def test_is_none_union(): + left = ak.Array([[[{"x": 1, "y": None}]]]) + right = ak.Array([[[{"a": 1, "b": None}]]]) + + array = ak.concatenate([left, right], axis=2) + + assert ak.is_none(array, axis=-1).tolist() == [ + [[{"x": False, "y": True}, {"a": False, "b": True}]] + ] From d73d78ace4106aeb58edd1198a097315ec2b273b Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 9 Dec 2022 15:28:16 +0000 Subject: [PATCH 26/26] Update tests/test_1137-num.py --- tests/test_1137-num.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_1137-num.py b/tests/test_1137-num.py index 809485bbf9..d595819097 100644 --- a/tests/test_1137-num.py +++ b/tests/test_1137-num.py @@ -40,6 +40,7 @@ def test_bytemaskedarray_num(): def test_emptyarray(): array = ak.contents.EmptyArray() assert to_list(ak.num(array, 0)) == 0 + assert to_list(ak.num(array, -1)) == 0 with pytest.raises(np.AxisError) as err: ak.num(array, 1) assert "axis=1 exceeds the depth" in str(err.value)