From 08cdbaaac9bfbd243042b23fc9ecfe3522e3fa4b Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 22 Oct 2024 21:39:06 +0200 Subject: [PATCH 1/8] Basic fix for ParamSpec with Generic --- src/typing_extensions.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/typing_extensions.py b/src/typing_extensions.py index c5e84b31..94a9f563 100644 --- a/src/typing_extensions.py +++ b/src/typing_extensions.py @@ -3006,6 +3006,10 @@ def wrapper(*args, **kwargs): f"a class or callable, not {arg!r}" ) +def _is_param_expr(arg): + return arg is ... or isinstance( + arg, (tuple, list, ParamSpec, _ConcatenateGenericAlias) + ) # We have to do some monkey patching to deal with the dual nature of # Unpack/TypeVarTuple: @@ -3020,6 +3024,18 @@ def _check_generic(cls, parameters, elen=_marker): This gives a nice error message in case of count mismatch. """ + if (inspect.isclass(cls) and issubclass(cls, typing.Generic) + and any(isinstance(t, ParamSpec) for t in cls.__parameters__) + ): + # should actually modify parameters but is immutable + if ( + len(cls.__parameters__) == 1 + and parameters + and not _is_param_expr(parameters[0]) + ): + assert isinstance(cls.__parameters__[0], ParamSpec) + return + if not elen: raise TypeError(f"{cls} is not a generic class") if elen is _marker: From 076ab7c4131d3b109dd6c1b441cc86e3c3d3d2be Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 22 Oct 2024 23:02:33 +0200 Subject: [PATCH 2/8] Add tests --- src/test_typing_extensions.py | 65 +++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index 1b43f90f..2f1178c3 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -3705,6 +3705,10 @@ def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: ... self.assertEqual(Y.__parameters__, ()) self.assertEqual(Y.__args__, ((int, str, str), bytes, memoryview)) + # Regression test; fixing #126 might cause an error here + with self.assertRaisesRegex(TypeError, "not a generic class"): + Y[int] + def test_protocol_generic_over_typevartuple(self): Ts = TypeVarTuple("Ts") T = TypeVar("T") @@ -5259,6 +5263,7 @@ class X(Generic[T, P]): class Y(Protocol[T, P]): pass + things = "arguments" if sys.version_info >= (3, 10) else "parameters" for klass in X, Y: with self.subTest(klass=klass.__name__): G1 = klass[int, P_2] @@ -5273,13 +5278,22 @@ class Y(Protocol[T, P]): self.assertEqual(G3.__args__, (int, Concatenate[int, ...])) self.assertEqual(G3.__parameters__, ()) + with self.assertRaisesRegex( + TypeError, + f"Too few {things} for {klass}" + ): + klass[int] + # The following are some valid uses cases in PEP 612 that don't work: # These do not work in 3.9, _type_check blocks the list and ellipsis. # G3 = X[int, [int, bool]] # G4 = X[int, ...] # G5 = Z[[int, str, bool]] - # Not working because this is special-cased in 3.10. - # G6 = Z[int, str, bool] + + def test_single_argument_generic(self): + P = ParamSpec("P") + T = TypeVar("T") + P_2 = ParamSpec("P_2") class Z(Generic[P]): pass @@ -5287,6 +5301,53 @@ class Z(Generic[P]): class ProtoZ(Protocol[P]): pass + things = "arguments" if sys.version_info >= (3, 10) else "parameters" + for klass in Z, ProtoZ: + with self.subTest(klass=klass.__name__): + # Note: For 3.10+ __args__ are nested tuples here ((int, ),) instead of (int, ) + G6 = klass[int, str, bool] + G6args = G6.__args__[0] if sys.version_info >= (3, 10) else G6.__args__ + self.assertEqual(G6args, (int, str, bool)) + self.assertEqual(G6.__parameters__, ()) + + # P = [int] + G7 = klass[int] + G7args = G7.__args__[0] if sys.version_info >= (3, 10) else G7.__args__ + self.assertEqual(G7args, (int,)) + self.assertEqual(G7.__parameters__, ()) + + # P = [int, int, ...] + G8 = klass[T, Concatenate[int, ...]] + G8args = G8.__args__[0] if sys.version_info >= (3, 10) else G8.__args__ + self.assertEqual(G8args, (T, Concatenate[int, ...])) + + # P = [int, int, P_2] + G9 = klass[int, Concatenate[T, P_2]] + G9args = G9.__args__[0] if sys.version_info >= (3, 10) else G9.__args__ + self.assertEqual(G9args, (int, Concatenate[T, P_2])) + + with self.subTest("Recursive ParamSpec Test A"): + if sys.version_info[:2] == (3, 10): + self.skipTest("Parameter detection fails in 3.10") + with self.assertRaisesRegex(TypeError, f"Too few {things}"): + G9[int] # for python 3.10 this has no parameters + self.assertEqual(G8.__parameters__, (T,)) + self.assertEqual(G9.__parameters__, (T, P_2)) + + with self.subTest("Recursive ParamSpec Test B"): + if sys.version_info[:2] == (3, 10): + self.skipTest("Parameter detection fails in 3.10") + if (3, 11, 0) <= sys.version_info[:3] < (3, 11, 3): + self.skipTest("Wrong recursive substitution") + H1 = G8[int] + H2 = G8[T][int] + self.assertEqual(H2.__parameters__, ()) + self.assertEqual(H1.__parameters__, ()) + with self.assertRaisesRegex(TypeError, "not a generic class"): + H1[str] # for python 3.11.0-3 this still has a parameter + with self.assertRaisesRegex(TypeError, "not a generic class"): + H2[str] # for python 3.11.0-3 this still has a parameter + def test_pickle(self): global P, P_co, P_contra, P_default P = ParamSpec('P') From 06e5cff90eaff6e5823deb0389e7516254a16898 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 23 Oct 2024 12:22:59 +0200 Subject: [PATCH 3/8] Modified tests --- src/test_typing_extensions.py | 53 +++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index 2f1178c3..aaf8b326 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -5305,10 +5305,9 @@ class ProtoZ(Protocol[P]): for klass in Z, ProtoZ: with self.subTest(klass=klass.__name__): # Note: For 3.10+ __args__ are nested tuples here ((int, ),) instead of (int, ) - G6 = klass[int, str, bool] + G6 = klass[int, str, T] G6args = G6.__args__[0] if sys.version_info >= (3, 10) else G6.__args__ - self.assertEqual(G6args, (int, str, bool)) - self.assertEqual(G6.__parameters__, ()) + self.assertEqual(G6args, (int, str, T)) # P = [int] G7 = klass[int] @@ -5316,38 +5315,62 @@ class ProtoZ(Protocol[P]): self.assertEqual(G7args, (int,)) self.assertEqual(G7.__parameters__, ()) - # P = [int, int, ...] - G8 = klass[T, Concatenate[int, ...]] - G8args = G8.__args__[0] if sys.version_info >= (3, 10) else G8.__args__ - self.assertEqual(G8args, (T, Concatenate[int, ...])) + G8 = klass[Concatenate[T, ...]] + self.assertEqual(G8.__args__, (Concatenate[T, ...], )) - # P = [int, int, P_2] - G9 = klass[int, Concatenate[T, P_2]] - G9args = G9.__args__[0] if sys.version_info >= (3, 10) else G9.__args__ - self.assertEqual(G9args, (int, Concatenate[T, P_2])) + G9 = klass[Concatenate[T, P_2]] + self.assertEqual(G9.__args__, (Concatenate[T, P_2], )) - with self.subTest("Recursive ParamSpec Test A"): + # This is an invalid form but useful for testing correct subsitution + G10 = klass[int, Concatenate[str, P]] + G10args = G10.__args__[0] if sys.version_info >= (3, 10) else G10.__args__ + self.assertEqual(G10args, (int, Concatenate[str, P], )) + + with self.subTest("Check generic substitution", klass=klass.__name__): + if sys.version_info < (3, 10): + self.skipTest("_ConcatenateGenericAlias not subscriptable") + with self.assertRaisesRegex(TypeError, "Expected a list of types, an ellipsis, ParamSpec, or Concatenate"): + G9[int, int] + + with self.subTest("Check list as parameter expression", klass=klass.__name__): + if sys.version_info < (3, 10): + self.skipTest("Cannot pass non-types") + G5 = klass[[int, str, T]] + self.assertEqual(G5.__args__, ((int, str, T),)) + H9 = G9[int, [T]] + + with self.subTest("Check parametrization", klass=klass.__name__): if sys.version_info[:2] == (3, 10): self.skipTest("Parameter detection fails in 3.10") with self.assertRaisesRegex(TypeError, f"Too few {things}"): G9[int] # for python 3.10 this has no parameters + self.assertEqual(G6.__parameters__, (T,)) self.assertEqual(G8.__parameters__, (T,)) self.assertEqual(G9.__parameters__, (T, P_2)) + if sys.version_info >= (3, 10): # skipped above + self.assertEqual(G5.__parameters__, (T,)) + self.assertEqual(H9.__parameters__, (T,)) - with self.subTest("Recursive ParamSpec Test B"): + with self.subTest("Check further substitution", klass=klass.__name__): + if sys.version_info < (3, 10): + self.skipTest("_ConcatenateGenericAlias not subscriptable") if sys.version_info[:2] == (3, 10): self.skipTest("Parameter detection fails in 3.10") if (3, 11, 0) <= sys.version_info[:3] < (3, 11, 3): self.skipTest("Wrong recursive substitution") H1 = G8[int] - H2 = G8[T][int] - self.assertEqual(H2.__parameters__, ()) self.assertEqual(H1.__parameters__, ()) with self.assertRaisesRegex(TypeError, "not a generic class"): H1[str] # for python 3.11.0-3 this still has a parameter + + H2 = G8[T][int] + self.assertEqual(H2.__parameters__, ()) with self.assertRaisesRegex(TypeError, "not a generic class"): H2[str] # for python 3.11.0-3 this still has a parameter + H3 = G10[int] + self.assertEqual(H3.__args__, ((int, (str, int)),)) + def test_pickle(self): global P, P_co, P_contra, P_default P = ParamSpec('P') From 4d2bdcb968f8f0a42b80f880f9192f3ac036107d Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 23 Oct 2024 12:50:02 +0200 Subject: [PATCH 4/8] split tests in two groups --- src/test_typing_extensions.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index aaf8b326..9fa721f6 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -5301,7 +5301,6 @@ class Z(Generic[P]): class ProtoZ(Protocol[P]): pass - things = "arguments" if sys.version_info >= (3, 10) else "parameters" for klass in Z, ProtoZ: with self.subTest(klass=klass.__name__): # Note: For 3.10+ __args__ are nested tuples here ((int, ),) instead of (int, ) @@ -5317,6 +5316,7 @@ class ProtoZ(Protocol[P]): G8 = klass[Concatenate[T, ...]] self.assertEqual(G8.__args__, (Concatenate[T, ...], )) + self.assertEqual(G8.__parameters__, (T,)) G9 = klass[Concatenate[T, P_2]] self.assertEqual(G9.__args__, (Concatenate[T, P_2], )) @@ -5326,6 +5326,25 @@ class ProtoZ(Protocol[P]): G10args = G10.__args__[0] if sys.version_info >= (3, 10) else G10.__args__ self.assertEqual(G10args, (int, Concatenate[str, P], )) + def test_single_argument_generic_with_parameter_expressions(self): + P = ParamSpec("P") + T = TypeVar("T") + P_2 = ParamSpec("P_2") + + class Z(Generic[P]): + pass + + class ProtoZ(Protocol[P]): + pass + + things = "arguments" if sys.version_info >= (3, 10) else "parameters" + for klass in Z, ProtoZ: + with self.subTest(klass=klass.__name__): + G6 = klass[int, str, T] + G8 = klass[Concatenate[T, ...]] + G9 = klass[Concatenate[T, P_2]] + G10 = klass[int, Concatenate[str, P]] + with self.subTest("Check generic substitution", klass=klass.__name__): if sys.version_info < (3, 10): self.skipTest("_ConcatenateGenericAlias not subscriptable") @@ -5339,14 +5358,13 @@ class ProtoZ(Protocol[P]): self.assertEqual(G5.__args__, ((int, str, T),)) H9 = G9[int, [T]] + self.assertEqual(G9.__parameters__, (T, P_2)) with self.subTest("Check parametrization", klass=klass.__name__): if sys.version_info[:2] == (3, 10): self.skipTest("Parameter detection fails in 3.10") with self.assertRaisesRegex(TypeError, f"Too few {things}"): G9[int] # for python 3.10 this has no parameters self.assertEqual(G6.__parameters__, (T,)) - self.assertEqual(G8.__parameters__, (T,)) - self.assertEqual(G9.__parameters__, (T, P_2)) if sys.version_info >= (3, 10): # skipped above self.assertEqual(G5.__parameters__, (T,)) self.assertEqual(H9.__parameters__, (T,)) From a280a44079841779659a374af2d246ee24b0d579 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 28 Oct 2024 11:15:06 +0100 Subject: [PATCH 5/8] Better comment and unified if statement --- src/typing_extensions.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/typing_extensions.py b/src/typing_extensions.py index 94a9f563..68462510 100644 --- a/src/typing_extensions.py +++ b/src/typing_extensions.py @@ -3024,17 +3024,16 @@ def _check_generic(cls, parameters, elen=_marker): This gives a nice error message in case of count mismatch. """ + # If substituting a single ParamSpec with multiple arguments + # we do not check the count if (inspect.isclass(cls) and issubclass(cls, typing.Generic) - and any(isinstance(t, ParamSpec) for t in cls.__parameters__) + and len(cls.__parameters__) == 1 + and isinstance(cls.__parameters__[0], ParamSpec) + and parameters + and not _is_param_expr(parameters[0]) ): - # should actually modify parameters but is immutable - if ( - len(cls.__parameters__) == 1 - and parameters - and not _is_param_expr(parameters[0]) - ): - assert isinstance(cls.__parameters__[0], ParamSpec) - return + # Generic modifies parameters variable, but here we cannot do this + return if not elen: raise TypeError(f"{cls} is not a generic class") From 5491d3ad15677ced524a20b80efd3efabbc6266c Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 27 Nov 2024 11:47:40 +0100 Subject: [PATCH 6/8] Restriction does not apply anymore with --- src/test_typing_extensions.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index 265d915b..d1fd6a7a 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -7567,11 +7567,9 @@ def test_callable_with_concatenate(self): self.assertEqual(callable_concat.__parameters__, (P2,)) concat_usage = callable_concat[str] with self.subTest("get_args of Concatenate in TypeAliasType"): - if not TYPING_3_9_0: + if not TYPING_3_10_0: # args are: ([, ~P2],) self.skipTest("Nested ParamSpec is not substituted") - if sys.version_info < (3, 10, 2): - self.skipTest("GenericAlias keeps Concatenate in __args__ prior to 3.10.2") self.assertEqual(get_args(concat_usage), ((int, str),)) with self.subTest("Equality of parameter_expression without []"): if not TYPING_3_10_0: From 6d31bbb20b3a48fd9544fcf75b37c386f5685c66 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 27 Nov 2024 17:28:33 +0100 Subject: [PATCH 7/8] test and correct _is_param_expr for typing.Concatenate --- src/test_typing_extensions.py | 21 +++++++++++++++++++++ src/typing_extensions.py | 21 +++++++++++++++++---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index d1fd6a7a..a211fd07 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -5326,6 +5326,16 @@ class ProtoZ(Protocol[P]): G10args = G10.__args__[0] if sys.version_info >= (3, 10) else G10.__args__ self.assertEqual(G10args, (int, Concatenate[str, P], )) + @skipUnless(TYPING_3_10_0, "ParamSpec not present before 3.10") + def test_is_param_expr(self): + P = ParamSpec("P") + P_typing = typing.ParamSpec("P_typing") + self.assertTrue(typing_extensions._is_param_expr(P)) + self.assertTrue(typing_extensions._is_param_expr(P_typing)) + if hasattr(typing, "_is_param_expr"): + self.assertTrue(typing._is_param_expr(P)) + self.assertTrue(typing._is_param_expr(P_typing)) + def test_single_argument_generic_with_parameter_expressions(self): P = ParamSpec("P") T = TypeVar("T") @@ -5571,6 +5581,17 @@ def test_eq(self): self.assertNotEqual(C4, C6) + @skipUnless(TYPING_3_10_0, "Concatenate not present before 3.10") + def test_is_param_expr(self): + P = ParamSpec('P') + concat = Concatenate[str, P] + typing_concat = typing.Concatenate[str, P] + self.assertTrue(typing_extensions._is_param_expr(concat)) + self.assertTrue(typing_extensions._is_param_expr(typing_concat)) + if hasattr(typing, "_is_param_expr"): + self.assertTrue(typing._is_param_expr(concat)) + self.assertTrue(typing._is_param_expr(typing_concat)) + class TypeGuardTests(BaseTestCase): def test_basics(self): TypeGuard[int] # OK diff --git a/src/typing_extensions.py b/src/typing_extensions.py index a79e35a2..13a19ed2 100644 --- a/src/typing_extensions.py +++ b/src/typing_extensions.py @@ -3006,10 +3006,23 @@ def wrapper(*args, **kwargs): f"a class or callable, not {arg!r}" ) -def _is_param_expr(arg): - return arg is ... or isinstance( - arg, (tuple, list, ParamSpec, _ConcatenateGenericAlias) - ) +if sys.version_info < (3, 10): + def _is_param_expr(arg): + return arg is ... or isinstance( + arg, (tuple, list, ParamSpec, _ConcatenateGenericAlias) + ) +else: + def _is_param_expr(arg): + return arg is ... or isinstance( + arg, + ( + tuple, + list, + ParamSpec, + _ConcatenateGenericAlias, + typing._ConcatenateGenericAlias, + ), + ) # We have to do some monkey patching to deal with the dual nature of # Unpack/TypeVarTuple: From e79fadbf25d5d387928e5284c126845463aac563 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 27 Nov 2024 18:49:00 +0100 Subject: [PATCH 8/8] verify _check_generic also for typing variants --- src/test_typing_extensions.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/test_typing_extensions.py b/src/test_typing_extensions.py index a211fd07..a8c5da25 100644 --- a/src/test_typing_extensions.py +++ b/src/test_typing_extensions.py @@ -5399,6 +5399,24 @@ class ProtoZ(Protocol[P]): H3 = G10[int] self.assertEqual(H3.__args__, ((int, (str, int)),)) + @skipUnless(TYPING_3_10_0, "ParamSpec not present before 3.10") + def test_substitution_with_typing_variants(self): + # verifies substitution and typing._check_generic working with typing variants + P = ParamSpec("P") + typing_P = typing.ParamSpec("typing_P") + typing_Concatenate = typing.Concatenate[int, P] + + class Z(Generic[typing_P]): + pass + + P1 = Z[typing_P] + self.assertEqual(P1.__parameters__, (typing_P,)) + self.assertEqual(P1.__args__, (typing_P,)) + + C1 = Z[typing_Concatenate] + self.assertEqual(C1.__parameters__, (P,)) + self.assertEqual(C1.__args__, (typing_Concatenate,)) + def test_pickle(self): global P, P_co, P_contra, P_default P = ParamSpec('P')