Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gh-114053: Fix another edge case involving get_type_hints, PEP 695 and PEP 563 #120272

Merged
merged 10 commits into from
Jun 25, 2024
66 changes: 62 additions & 4 deletions Lib/test/test_typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -4858,20 +4858,30 @@ def f(x: X): ...
{'x': list[list[ForwardRef('X')]]}
)

def test_pep695_generic_with_future_annotations(self):
def test_pep695_generic_class_with_future_annotations(self):
original_globals = dict(ann_module695.__dict__)

hints_for_A = get_type_hints(ann_module695.A)
A_type_params = ann_module695.A.__type_params__
self.assertIs(hints_for_A["x"], A_type_params[0])
self.assertEqual(hints_for_A["y"].__args__[0], Unpack[A_type_params[1]])
self.assertIs(hints_for_A["z"].__args__[0], A_type_params[2])

# should not have changed as a result of the get_type_hints() calls!
self.assertEqual(ann_module695.__dict__, original_globals)

def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self):
hints_for_B = get_type_hints(ann_module695.B)
self.assertEqual(hints_for_B.keys(), {"x", "y", "z"})
self.assertEqual(hints_for_B, {"x": int, "y": str, "z": bytes})

def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self):
hints_for_C = get_type_hints(ann_module695.C)
self.assertEqual(
set(hints_for_B.values()) ^ set(ann_module695.B.__type_params__),
set()
set(hints_for_C.values()),
set(ann_module695.C.__type_params__)
)

def test_pep_695_generic_function_with_future_annotations(self):
hints_for_generic_function = get_type_hints(ann_module695.generic_function)
func_t_params = ann_module695.generic_function.__type_params__
self.assertEqual(
Expand All @@ -4882,6 +4892,54 @@ def test_pep695_generic_with_future_annotations(self):
self.assertIs(hints_for_generic_function["z"].__origin__, func_t_params[2])
self.assertIs(hints_for_generic_function["zz"].__origin__, func_t_params[2])

def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self):
self.assertEqual(
set(get_type_hints(ann_module695.generic_function_2).values()),
set(ann_module695.generic_function_2.__type_params__)
)

def test_pep_695_generic_method_with_future_annotations(self):
hints_for_generic_method = get_type_hints(ann_module695.D.generic_method)
params = {
param.__name__: param
for param in ann_module695.D.generic_method.__type_params__
}
self.assertEqual(
hints_for_generic_method,
{"x": params["Foo"], "y": params["Bar"], "return": types.NoneType}
)

def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self):
self.assertEqual(
set(get_type_hints(ann_module695.D.generic_method_2).values()),
set(ann_module695.D.generic_method_2.__type_params__)
)

def test_pep_695_generics_with_future_annotations_nested_in_function(self):
results = ann_module695.nested()

self.assertEqual(
set(results.hints_for_E.values()),
set(results.E.__type_params__)
)
self.assertEqual(
set(results.hints_for_E_meth.values()),
set(results.E.generic_method.__type_params__)
)
self.assertNotEqual(
set(results.hints_for_E_meth.values()),
set(results.E.__type_params__)
)
self.assertEqual(
set(results.hints_for_E_meth.values()).intersection(results.E.__type_params__),
set()
)

self.assertEqual(
set(results.hints_for_generic_func.values()),
set(results.generic_func.__type_params__)
)

def test_extended_generic_rules_subclassing(self):
class T1(Tuple[T, KT]): ...
class T2(Tuple[T, ...]): ...
Expand Down
50 changes: 50 additions & 0 deletions Lib/test/typinganndata/ann_module695.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,56 @@ class B[T, *Ts, **P]:
z: P


Eggs = int
Spam = str


class C[Eggs, **Spam]:
x: Eggs
y: Spam


def generic_function[T, *Ts, **P](
x: T, *y: *Ts, z: P.args, zz: P.kwargs
) -> None: ...


def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass


class D:
Foo = int
Bar = str

def generic_method[Foo, **Bar](
self, x: Foo, y: Bar
) -> None: ...

def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass


def nested():
from types import SimpleNamespace
from typing import get_type_hints

Eggs = bytes
Spam = memoryview


class E[Eggs, **Spam]:
x: Eggs
y: Spam

def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass


def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass


return SimpleNamespace(
E=E,
hints_for_E=get_type_hints(E),
hints_for_E_meth=get_type_hints(E.generic_method),
generic_func=generic_function,
hints_for_generic_func=get_type_hints(generic_function)
)
23 changes: 16 additions & 7 deletions Lib/typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -1060,15 +1060,24 @@ def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard
globalns = getattr(
sys.modules.get(self.__forward_module__, None), '__dict__', globalns
)

# type parameters require some special handling,
# as they exist in their own scope
# but `eval()` does not have a dedicated parameter for that scope.
# For classes, names in type parameter scopes should override
# names in the global scope (which here are called `localns`!),
# but should in turn be overridden by names in the class scope
# (which here are called `globalns`!)
if type_params:
# "Inject" type parameters into the local namespace
# (unless they are shadowed by assignments *in* the local namespace),
# as a way of emulating annotation scopes when calling `eval()`
locals_to_pass = {param.__name__: param for param in type_params} | localns
else:
locals_to_pass = localns
globalns, localns = dict(globalns), dict(localns)
for param in type_params:
param_name = param.__name__
if not self.__forward_is_class__ or param_name not in globalns:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need the __forward_is_class__ condition? I don't understand why class-scoped forwardrefs are special here.

Copy link
Member Author

@AlexWaygood AlexWaygood Jun 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For functions, names in type_params should always take highest priority, and this is implemented by inserting the type params into the local namespace (which is here called globalns) and popping them from the global namespace (which is here called localns). For classes, we only want to give type_params highest priority if they haven't been overridden in the local scope, however.

globalns[param_name] = param
localns.pop(param_name, None)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need to do this? Shouldn't the local namespace override the type params?

Test case:

class X[T]:
    T = int
    x: T

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I already have that test case included in this PR, and it fails unless we have this block of code here. The local namespace shoudl indeed override the type param, but the locals here are found in the globalns, and the globals here are found in the localns variable. This highly confusing situation is because the globals and locals are swapped higher up in the call stack here, for backwards compatibility reasons if you believe the comment:

cpython/Lib/typing.py

Lines 2419 to 2426 in a3711af

if localns is None and globalns is None:
# This is surprising, but required. Before Python 3.10,
# get_type_hints only evaluated the globalns of
# a class. To maintain backwards compatibility, we reverse
# the globalns and localns order so that eval() looks into
# *base_globals* first rather than *base_locals*.
# This only affects ForwardRefs.
base_globals, base_locals = base_locals, base_globals

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I saw that comment, maybe we should find a way to get rid of that compatibility hack.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really see a way round it, short of deprecating get_type_hints altogether. Which... I'm not 100% against, but it would be quite disruptive.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There could be other approaches, like adding new keyword arguments. Will have to think about it more.

Copy link
Contributor

@picnixz picnixz Jun 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you decide to add keywords arguments or deprecate things, please tell me in advance so that I can be reactive on Sphinx' side if needed.


type_ = _type_check(
eval(self.__forward_code__, globalns, locals_to_pass),
eval(self.__forward_code__, globalns, localns),
"Forward references must evaluate to types.",
is_argument=self.__forward_is_argument__,
allow_special_forms=self.__forward_is_class__,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fix edge-case bug where :func:`typing.get_type_hints` would produce
incorrect results if type parameters in a class scope were overridden by
assignments in a class scope and ``from __future__ import annotations``
semantics were enabled. Patch by Alex Waygood.
Loading