Skip to content

Commit

Permalink
BUG: rank raising for arrow string dtypes (#55362)
Browse files Browse the repository at this point in the history
  • Loading branch information
phofl authored Oct 4, 2023
1 parent 6b84daa commit 6a83910
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 7 deletions.
1 change: 1 addition & 0 deletions doc/source/whatsnew/v2.1.2.rst
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ Bug fixes
- Fixed bug in :meth:`DataFrame.resample` not respecting ``closed`` and ``label`` arguments for :class:`~pandas.tseries.offsets.BusinessDay` (:issue:`55282`)
- Fixed bug in :meth:`DataFrame.resample` where bin edges were not correct for :class:`~pandas.tseries.offsets.BusinessDay` (:issue:`55281`)
- Fixed bug in :meth:`Index.insert` raising when inserting ``None`` into :class:`Index` with ``dtype="string[pyarrow_numpy]"`` (:issue:`55365`)
- Fixed bug in :meth:`Series.rank` for ``string[pyarrow_numpy]`` dtype (:issue:`55362`)
- Silence ``Period[B]`` warnings introduced by :issue:`53446` during normal plotting activity (:issue:`55138`)
-

Expand Down
31 changes: 25 additions & 6 deletions pandas/core/arrays/arrow/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -1747,7 +1747,7 @@ def __setitem__(self, key, value) -> None:
data = pa.chunked_array([data])
self._pa_array = data

def _rank(
def _rank_calc(
self,
*,
axis: AxisInt = 0,
Expand All @@ -1756,9 +1756,6 @@ def _rank(
ascending: bool = True,
pct: bool = False,
):
"""
See Series.rank.__doc__.
"""
if pa_version_under9p0 or axis != 0:
ranked = super()._rank(
axis=axis,
Expand All @@ -1773,7 +1770,7 @@ def _rank(
else:
pa_type = pa.uint64()
result = pa.array(ranked, type=pa_type, from_pandas=True)
return type(self)(result)
return result

data = self._pa_array.combine_chunks()
sort_keys = "ascending" if ascending else "descending"
Expand Down Expand Up @@ -1812,7 +1809,29 @@ def _rank(
divisor = pc.count(result)
result = pc.divide(result, divisor)

return type(self)(result)
return result

def _rank(
self,
*,
axis: AxisInt = 0,
method: str = "average",
na_option: str = "keep",
ascending: bool = True,
pct: bool = False,
):
"""
See Series.rank.__doc__.
"""
return type(self)(
self._rank_calc(
axis=axis,
method=method,
na_option=na_option,
ascending=ascending,
pct=pct,
)
)

def _quantile(self, qs: npt.NDArray[np.float64], interpolation: str) -> Self:
"""
Expand Down
28 changes: 27 additions & 1 deletion pandas/core/arrays/string_arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
from collections.abc import Sequence

from pandas._typing import (
AxisInt,
Dtype,
Scalar,
npt,
Expand Down Expand Up @@ -501,6 +502,28 @@ def _str_find(self, sub: str, start: int = 0, end: int | None = None):
def _convert_int_dtype(self, result):
return Int64Dtype().__from_arrow__(result)

def _rank(
self,
*,
axis: AxisInt = 0,
method: str = "average",
na_option: str = "keep",
ascending: bool = True,
pct: bool = False,
):
"""
See Series.rank.__doc__.
"""
return self._convert_int_dtype(
self._rank_calc(
axis=axis,
method=method,
na_option=na_option,
ascending=ascending,
pct=pct,
)
)


class ArrowStringArrayNumpySemantics(ArrowStringArray):
_storage = "pyarrow_numpy"
Expand Down Expand Up @@ -584,7 +607,10 @@ def _str_map(
return lib.map_infer_mask(arr, f, mask.view("uint8"))

def _convert_int_dtype(self, result):
result = result.to_numpy()
if isinstance(result, pa.Array):
result = result.to_numpy(zero_copy_only=False)
else:
result = result.to_numpy()
if result.dtype == np.int32:
result = result.astype(np.int64)
return result
Expand Down
12 changes: 12 additions & 0 deletions pandas/tests/frame/methods/test_rank.py
Original file line number Diff line number Diff line change
Expand Up @@ -488,3 +488,15 @@ def test_rank_mixed_axis_zero(self, data, expected):
df.rank()
result = df.rank(numeric_only=True)
tm.assert_frame_equal(result, expected)

@pytest.mark.parametrize(
"dtype, exp_dtype",
[("string[pyarrow]", "Int64"), ("string[pyarrow_numpy]", "float64")],
)
def test_rank_string_dtype(self, dtype, exp_dtype):
# GH#55362
pytest.importorskip("pyarrow")
obj = Series(["foo", "foo", None, "foo"], dtype=dtype)
result = obj.rank(method="first")
expected = Series([1, 2, None, 3], dtype=exp_dtype)
tm.assert_series_equal(result, expected)

0 comments on commit 6a83910

Please sign in to comment.