Skip to content

Commit

Permalink
Backport #59144 on 2.2.x / 2.3.x (remove ops div class to solve #2137) (
Browse files Browse the repository at this point in the history
#59535)

* remove core.computation.ops.Div resolves #21374 #58748

* need to preserve order

* updating tests

* (update whatsnew -- no whatsnew for 2.2.x and 2.3 yet)

* solve mypy issue

* fixing pytests

* better than cast

* adding specific test

(* Update pandas/tests/frame/test_query_eval.py // Not backported, fails on 2.2)



* Update pandas/tests/computation/test_eval.py



---------

Co-authored-by: Laurent Mutricy <[email protected]>
Co-authored-by: Matthew Roeschke <[email protected]>
Co-authored-by: Thomas Li <[email protected]>
  • Loading branch information
4 people authored Sep 18, 2024
1 parent 4a20adb commit 2127b42
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 60 deletions.
1 change: 1 addition & 0 deletions pandas/_testing/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@

COMPLEX_DTYPES: list[Dtype] = [complex, "complex64", "complex128"]
STRING_DTYPES: list[Dtype] = [str, "str", "U"]
COMPLEX_FLOAT_DTYPES: list[Dtype] = [*COMPLEX_DTYPES, *FLOAT_NUMPY_DTYPES]

DATETIME64_DTYPES: list[Dtype] = ["datetime64[ns]", "M8[ns]"]
TIMEDELTA64_DTYPES: list[Dtype] = ["timedelta64[ns]", "m8[ns]"]
Expand Down
15 changes: 15 additions & 0 deletions pandas/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -1403,6 +1403,21 @@ def complex_dtype(request):
return request.param


@pytest.fixture(params=tm.COMPLEX_FLOAT_DTYPES)
def complex_or_float_dtype(request):
"""
Parameterized fixture for complex and numpy float dtypes.
* complex
* 'complex64'
* 'complex128'
* float
* 'float32'
* 'float64'
"""
return request.param


@pytest.fixture(params=tm.SIGNED_INT_NUMPY_DTYPES)
def any_signed_int_numpy_dtype(request):
"""
Expand Down
6 changes: 1 addition & 5 deletions pandas/core/computation/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
UNARY_OPS_SYMS,
BinOp,
Constant,
Div,
FuncNode,
Op,
Term,
Expand Down Expand Up @@ -370,7 +369,7 @@ class BaseExprVisitor(ast.NodeVisitor):
"Add",
"Sub",
"Mult",
None,
"Div",
"Pow",
"FloorDiv",
"Mod",
Expand Down Expand Up @@ -533,9 +532,6 @@ def visit_BinOp(self, node, **kwargs):
left, right = self._maybe_downcast_constants(left, right)
return self._maybe_evaluate_binop(op, op_class, left, right)

def visit_Div(self, node, **kwargs):
return lambda lhs, rhs: Div(lhs, rhs)

def visit_UnaryOp(self, node, **kwargs):
op = self.visit(node.op)
operand = self.visit(node.operand)
Expand Down
49 changes: 0 additions & 49 deletions pandas/core/computation/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -332,31 +332,6 @@ def _not_in(x, y):
_binary_ops_dict.update(d)


def _cast_inplace(terms, acceptable_dtypes, dtype) -> None:
"""
Cast an expression inplace.
Parameters
----------
terms : Op
The expression that should cast.
acceptable_dtypes : list of acceptable numpy.dtype
Will not cast if term's dtype in this list.
dtype : str or numpy.dtype
The dtype to cast to.
"""
dt = np.dtype(dtype)
for term in terms:
if term.type in acceptable_dtypes:
continue

try:
new_value = term.value.astype(dt)
except AttributeError:
new_value = dt.type(term.value)
term.update(new_value)


def is_term(obj) -> bool:
return isinstance(obj, Term)

Expand Down Expand Up @@ -517,30 +492,6 @@ def isnumeric(dtype) -> bool:
return issubclass(np.dtype(dtype).type, np.number)


class Div(BinOp):
"""
Div operator to special case casting.
Parameters
----------
lhs, rhs : Term or Op
The Terms or Ops in the ``/`` expression.
"""

def __init__(self, lhs, rhs) -> None:
super().__init__("/", lhs, rhs)

if not isnumeric(lhs.return_type) or not isnumeric(rhs.return_type):
raise TypeError(
f"unsupported operand type(s) for {self.op}: "
f"'{lhs.return_type}' and '{rhs.return_type}'"
)

# do not upcast float32s to float64 un-necessarily
acceptable_dtypes = [np.float32, np.float64]
_cast_inplace(com.flatten(self), acceptable_dtypes, np.float64)


UNARY_OPS_SYMS = ("+", "-", "~", "not")
_unary_ops_funcs = (operator.pos, operator.neg, operator.invert, operator.invert)
_unary_ops_dict = dict(zip(UNARY_OPS_SYMS, _unary_ops_funcs))
Expand Down
22 changes: 16 additions & 6 deletions pandas/tests/computation/test_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -747,16 +747,26 @@ class TestTypeCasting:
@pytest.mark.parametrize("op", ["+", "-", "*", "**", "/"])
# maybe someday... numexpr has too many upcasting rules now
# chain(*(np.core.sctypes[x] for x in ['uint', 'int', 'float']))
@pytest.mark.parametrize("dt", [np.float32, np.float64])
@pytest.mark.parametrize("left_right", [("df", "3"), ("3", "df")])
def test_binop_typecasting(self, engine, parser, op, dt, left_right):
df = DataFrame(np.random.default_rng(2).standard_normal((5, 3)), dtype=dt)
def test_binop_typecasting(
self, engine, parser, op, complex_or_float_dtype, left_right, request
):
# GH#21374
dtype = complex_or_float_dtype
df = DataFrame(np.random.default_rng(2).standard_normal((5, 3)), dtype=dtype)
left, right = left_right
s = f"{left} {op} {right}"
res = pd.eval(s, engine=engine, parser=parser)
assert df.values.dtype == dt
assert res.values.dtype == dt
tm.assert_frame_equal(res, eval(s))
if dtype == "complex64" and engine == "numexpr":
mark = pytest.mark.xfail(
reason="numexpr issue with complex that are upcast "
"to complex 128 "
"https://github.com/pydata/numexpr/issues/492"
)
request.applymarker(mark)
assert df.values.dtype == dtype
assert res.values.dtype == dtype
tm.assert_frame_equal(res, eval(s), check_exact=False)


# -------------------------------------
Expand Down

0 comments on commit 2127b42

Please sign in to comment.