Use bitmask for MaskedArray mask

ci/deps/actions-310-minimum_versions.yaml

@@ -60,6 +60,7 @@ dependencies:
   - zstandard=0.19.0
 
   - pip:
+    - git+https://github.com/WillAyd/pandas-bitmask.git
     - adbc-driver-postgresql==0.10.0
     - adbc-driver-sqlite==0.8.0
     - tzdata==2022.7

ci/deps/actions-310.yaml

@@ -58,6 +58,7 @@ dependencies:
   - zstandard>=0.19.0
 
   - pip:
+    - git+https://github.com/WillAyd/pandas-bitmask.git
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7

ci/deps/actions-311-downstream_compat.yaml

@@ -72,6 +72,7 @@ dependencies:
   - pyyaml
   - py
   - pip:
+    - git+https://github.com/WillAyd/pandas-bitmask.git
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7

ci/deps/actions-311-numpydev.yaml

@@ -21,6 +21,7 @@ dependencies:
   - pip
 
   - pip:
+    - git+https://github.com/WillAyd/pandas-bitmask.git
     - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
     - "--pre"
     - "numpy"

ci/deps/actions-311-pyarrownightly.yaml

@@ -22,6 +22,7 @@ dependencies:
   - pip
 
   - pip:
+    - git+https://github.com/WillAyd/pandas-bitmask.git
     - "tzdata>=2022.7"
     - "--extra-index-url https://pypi.fury.io/arrow-nightlies/"
     - "--prefer-binary"

ci/deps/actions-311.yaml

@@ -58,5 +58,6 @@ dependencies:
   - zstandard>=0.19.0
 
   - pip:
+    - git+https://github.com/WillAyd/pandas-bitmask.git
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0

ci/deps/actions-312.yaml

@@ -58,6 +58,7 @@ dependencies:
   - zstandard>=0.19.0
 
   - pip:
+    - git+https://github.com/WillAyd/pandas-bitmask.git
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7

ci/deps/actions-pypy-39.yaml

@@ -23,4 +23,5 @@ dependencies:
   - numpy
   - python-dateutil
   - pip:
+    - git+https://github.com/WillAyd/pandas-bitmask.git
     - tzdata>=2022.7

ci/deps/circle-311-arm64.yaml

@@ -57,5 +57,6 @@ dependencies:
   - xlsxwriter>=3.0.5
   - zstandard>=0.19.0
   - pip:
+    - git+https://github.com/WillAyd/pandas-bitmask.git
     - adbc-driver-postgresql>=0.8.0
     - adbc-driver-sqlite>=0.8.0

environment.yml

@@ -116,6 +116,7 @@ dependencies:
   - pygments # Code highlighting
 
   - pip:
+      - git+https://github.com/WillAyd/pandas-bitmask.git
       - adbc-driver-postgresql>=0.10.0
       - adbc-driver-sqlite>=0.8.0
       - typing_extensions; python_version<"3.11"

pandas/_libs/index.pyx

@@ -140,7 +140,7 @@ cdef class IndexEngine:
 
     cdef readonly:
         ndarray values
-        ndarray mask
+        object mask
         HashTable mapping
         bint over_size_threshold
 

pandas/core/algorithms.py

@@ -16,6 +16,7 @@
 import warnings
 
 import numpy as np
+from pandas_mask import PandasMaskArray
 
 from pandas._libs import (
     algos,
@@ -1173,6 +1174,8 @@ def take(
     ... )
     array([ 10,  10, -10])
     """
+    if isinstance(arr, PandasMaskArray):  # TODO: implement take directly on mask
+        arr = np.array(arr)
     if not isinstance(
         arr,
         (np.ndarray, ABCExtensionArray, ABCIndex, ABCSeries, ABCNumpyExtensionArray),

pandas/core/arrays/masked.py

@@ -10,6 +10,7 @@
 import warnings
 
 import numpy as np
+from pandas_mask import PandasMaskArray
 
 from pandas._libs import (
     lib,
@@ -112,20 +113,23 @@ class BaseMaskedArray(OpsMixin, ExtensionArray):
 
     # our underlying data and mask are each ndarrays
     _data: np.ndarray
-    _mask: npt.NDArray[np.bool_]
+    _mask: PandasMaskArray
 
     @classmethod
     def _simple_new(cls, values: np.ndarray, mask: npt.NDArray[np.bool_]) -> Self:
         result = BaseMaskedArray.__new__(cls)
         result._data = values
-        result._mask = mask
+        result._mask = PandasMaskArray(mask)
         return result
 
     def __init__(
         self, values: np.ndarray, mask: npt.NDArray[np.bool_], copy: bool = False
     ) -> None:
         # values is supposed to already be validated in the subclass
-        if not (isinstance(mask, np.ndarray) and mask.dtype == np.bool_):
+        if not (
+            (isinstance(mask, np.ndarray) and mask.dtype == np.bool_)
+            or isinstance(mask, PandasMaskArray)
+        ):
             raise TypeError(
                 "mask should be boolean numpy array. Use "
                 "the 'pd.array' function instead"
@@ -678,7 +682,7 @@ def __arrow_array__(self, type=None):
         """
         import pyarrow as pa
 
-        return pa.array(self._data, mask=self._mask, type=type)
+        return pa.array(self._data, mask=np.asarray(self._mask), type=type)
 
     @property
     def _hasna(self) -> bool:

pandas/core/groupby/ops.py

@@ -324,7 +324,7 @@ def _cython_op_ndim_compat(
             # expand to 2d, dispatch, then squeeze if appropriate
             values2d = values[None, :]
             if mask is not None:
-                mask = mask[None, :]
+                mask = np.array(mask)[None, :]
             if result_mask is not None:
                 result_mask = result_mask[None, :]
             res = self._call_cython_op(

requirements-dev.txt

@@ -84,6 +84,7 @@ feedparser
 pyyaml
 requests
 pygments
+git+https://github.com/WillAyd/pandas-bitmask.git
 adbc-driver-postgresql>=0.10.0
 adbc-driver-sqlite>=0.8.0
 typing_extensions; python_version<"3.11"