From 7eabfe24356d056657305bc81093312d9f7cc78c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 14 Apr 2022 16:09:11 +0100 Subject: [PATCH 1/8] Add namespacing to class type variables to avoid conflicts --- mypy/semanal.py | 3 ++- mypy/tvar_scope.py | 14 +++++++++----- mypy/types.py | 12 +++++++++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 44ece0674732..1a54ffb1d1e2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1116,7 +1116,8 @@ def check_decorated_function_is_method(self, decorator: str, def visit_class_def(self, defn: ClassDef) -> None: self.statement = defn self.incomplete_type_stack.append(not defn.info) - with self.tvar_scope_frame(self.tvar_scope.class_frame()): + namespace = self.qualified_name(defn.name) + with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): self.analyze_class(defn) self.incomplete_type_stack.pop() diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index 0d8be7845e52..14d10714942b 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -1,5 +1,5 @@ from typing import Optional, Dict, Union -from mypy.types import TypeVarLikeType, TypeVarType, ParamSpecType, ParamSpecFlavor +from mypy.types import TypeVarLikeType, TypeVarType, ParamSpecType, ParamSpecFlavor, TypeVarId from mypy.nodes import ParamSpecExpr, TypeVarExpr, TypeVarLikeExpr, SymbolTableNode @@ -12,7 +12,8 @@ class TypeVarLikeScope: def __init__(self, parent: 'Optional[TypeVarLikeScope]' = None, is_class_scope: bool = False, - prohibited: 'Optional[TypeVarLikeScope]' = None) -> None: + prohibited: 'Optional[TypeVarLikeScope]' = None, + namespace: str = '') -> None: """Initializer for TypeVarLikeScope Parameters: @@ -27,6 +28,7 @@ def __init__(self, self.class_id = 0 self.is_class_scope = is_class_scope self.prohibited = prohibited + self.namespace = namespace if parent is not None: self.func_id = parent.func_id self.class_id = parent.class_id @@ -51,22 +53,24 @@ def method_frame(self) -> 'TypeVarLikeScope': """A new scope frame for binding a method""" return TypeVarLikeScope(self, False, None) - def class_frame(self) -> 'TypeVarLikeScope': + def class_frame(self, namespace: str) -> 'TypeVarLikeScope': """A new scope frame for binding a class. Prohibits *this* class's tvars""" - return TypeVarLikeScope(self.get_function_scope(), True, self) + return TypeVarLikeScope(self.get_function_scope(), True, self, namespace=namespace) def bind_new(self, name: str, tvar_expr: TypeVarLikeExpr) -> TypeVarLikeType: if self.is_class_scope: self.class_id += 1 i = self.class_id + namespace = self.namespace else: self.func_id -= 1 i = self.func_id + namespace = '' if isinstance(tvar_expr, TypeVarExpr): tvar_def: TypeVarLikeType = TypeVarType( name, tvar_expr.fullname, - i, + TypeVarId(i, namespace=namespace), values=tvar_expr.values, upper_bound=tvar_expr.upper_bound, variance=tvar_expr.variance, diff --git a/mypy/types.py b/mypy/types.py index b42335096198..5af8efa0ba78 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -416,9 +416,14 @@ class TypeVarId: # Class variable used for allocating fresh ids for metavariables. next_raw_id: ClassVar[int] = 1 - def __init__(self, raw_id: int, meta_level: int = 0) -> None: + # Fullname of class or function which declares this type variable + # (not the fullname of the TypeVar definition!), or '' + namespace: str + + def __init__(self, raw_id: int, meta_level: int = 0, *, namespace: str = '') -> None: self.raw_id = raw_id self.meta_level = meta_level + self.namespace = namespace @staticmethod def new(meta_level: int) -> 'TypeVarId': @@ -432,7 +437,8 @@ def __repr__(self) -> str: def __eq__(self, other: object) -> bool: if isinstance(other, TypeVarId): return (self.raw_id == other.raw_id and - self.meta_level == other.meta_level) + self.meta_level == other.meta_level and + self.namespace == other.namespace) else: return False @@ -440,7 +446,7 @@ def __ne__(self, other: object) -> bool: return not (self == other) def __hash__(self) -> int: - return hash((self.raw_id, self.meta_level)) + return hash((self.raw_id, self.meta_level, self.namespace)) def is_meta_var(self) -> bool: return self.meta_level > 0 From 4990c9b972d254e4e05c35ab8a760955ce9664ed Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 14 Apr 2022 16:48:33 +0100 Subject: [PATCH 2/8] Fix serialization --- mypy/types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/types.py b/mypy/types.py index 5af8efa0ba78..b91d8c415758 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -520,6 +520,7 @@ def serialize(self) -> JsonDict: 'name': self.name, 'fullname': self.fullname, 'id': self.id.raw_id, + 'namespace': self.id.namespace, 'values': [v.serialize() for v in self.values], 'upper_bound': self.upper_bound.serialize(), 'variance': self.variance, @@ -531,7 +532,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeVarType': return TypeVarType( data['name'], data['fullname'], - data['id'], + TypeVarId(data['id'], data['namespace']), [deserialize_type(v) for v in data['values']], deserialize_type(data['upper_bound']), data['variance'], From 0959c4af96ad47a41b41e9484781b6a78465039e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 14 Apr 2022 14:32:56 +0100 Subject: [PATCH 3/8] WIP tests --- test-data/unit/check-selftype.test | 89 +++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index b59c22dfae06..61c101fa03ae 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -893,11 +893,14 @@ from typing import Generic, TypeVar, Tuple T = TypeVar('T') S = TypeVar('S') U = TypeVar('U') +V = TypeVar('V') class C(Generic[T]): def magic(self: C[Tuple[S, U]]) -> Tuple[T, S, U]: ... -reveal_type(C[Tuple[int, str]]().magic()) # N: Revealed type is "Tuple[Tuple[builtins.int, builtins.str], builtins.int, builtins.str]" +class D(Generic[V]): + def f(self) -> None: + reveal_type(C[Tuple[V, str]]().magic()) # N: Revealed type is "Tuple[Tuple[V`1, builtins.str], V`1, builtins.str]" [builtins fixtures/tuple.pyi] [case testSelfTypeOnUnion] @@ -1167,3 +1170,87 @@ def build_wrapper_non_gen(descriptor: Descriptor[int]) -> BaseWrapper[str]: def build_sub_wrapper_non_gen(descriptor: Descriptor[int]) -> SubWrapper[str]: return SubWrapper.create_wrapper(descriptor) # E: Argument 1 to "create_wrapper" of "BaseWrapper" has incompatible type "Descriptor[int]"; expected "Descriptor[str]" [builtins fixtures/classmethod.pyi] + +[case testSelfTypeXXX] +from typing import TypeVar, Generic, Iterator, List, Tuple + +_T_co = TypeVar("_T_co", covariant=True) +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +S = TypeVar("S") + +class Z(Iterator[_T_co]): + def __new__(cls, + __iter1: List[_T1], + __iter2: List[_T2]) -> Z[Tuple[_T1, _T2]]: ... + def __iter__(self: S) -> S: ... + def __next__(self) -> _T_co: ... + +T = TypeVar('T') + +class C(Generic[T]): + a: List[T] + b: List[str] + + def f(self) -> None: + for x, y in Z(self.a, self.b): + reveal_type((x, y)) # N: Revealed type is "Tuple[T`1, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testSelfTypeYYY] +from typing import TypeVar, Generic, Iterator, List, Tuple + +_T_co = TypeVar("_T_co", covariant=True) +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +S = TypeVar("S") + +class Z(Iterator[_T_co]): + def __new__(cls, + __iter1: List[_T1], + __iter2: List[_T2]) -> Z[Tuple[_T1, _T2]]: ... + def __iter__(self) -> Z[_T_co]: ... + def __next__(self) -> _T_co: ... + +T = TypeVar('T') + +class C(Generic[T]): + a: List[T] + b: List[str] + + def f(self) -> None: + for x, y in Z(self.a, self.b): + reveal_type((x, y)) # N: Revealed type is "Tuple[T`1, builtins.str]" +[builtins fixtures/tuple.pyi] + +[case testSelfTypeZZZ] +from typing import TypeVar, Generic, Tuple + +_T_co = TypeVar("_T_co", covariant=True) +S = TypeVar("S") + +class Z(Generic[_T_co]): + def m(self: S) -> S: ... + +T = TypeVar('T') + +class C(Generic[T]): + def f(self, z: Z[Tuple[T, str]]) -> None: + reveal_type(z.m()) # N: Revealed type is "__main__.Z[Tuple[T`1, builtins.str]]" +[builtins fixtures/tuple.pyi] + +[case testSelfTypeCCC] +from typing import Generic, TypeVar, Tuple + +T = TypeVar('T') +S = TypeVar('S') +U = TypeVar('U') +V = TypeVar('V') + +class C(Generic[T]): + def magic(self: C[Tuple[S, U]]) -> Tuple[T, S, U]: ... + +class D(Generic[V]): + def f(self) -> None: + reveal_type(C[Tuple[V, str]]().magic()) # N: Revealed type is "Tuple[Tuple[V`1, builtins.str], V`1, builtins.str]" +[builtins fixtures/tuple.pyi] From f111aad3d788c0126e54120571159a4d47befd1a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 14 Apr 2022 17:17:26 +0100 Subject: [PATCH 4/8] Fix match statement issue --- mypy/checkpattern.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index 0fedec24cc37..e1d4f9fe285e 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -396,8 +396,7 @@ def visit_mapping_pattern(self, o: MappingPattern) -> PatternType: if is_subtype(current_type, mapping) and isinstance(current_type, Instance): mapping_inst = map_instance_to_supertype(current_type, mapping.type) dict_typeinfo = self.chk.lookup_typeinfo("builtins.dict") - dict_type = fill_typevars(dict_typeinfo) - rest_type = expand_type_by_instance(dict_type, mapping_inst) + rest_type = Instance(dict_typeinfo, mapping_inst.args) else: object_type = self.chk.named_type("builtins.object") rest_type = self.chk.named_generic_type("builtins.dict", From 14a24c00298632063d8552e11f23b5da0d1bfebf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 14 Apr 2022 17:20:36 +0100 Subject: [PATCH 5/8] Fix serialization --- mypy/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/types.py b/mypy/types.py index b91d8c415758..bcf848c477c2 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -532,7 +532,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeVarType': return TypeVarType( data['name'], data['fullname'], - TypeVarId(data['id'], data['namespace']), + TypeVarId(data['id'], namespace=data['namespace']), [deserialize_type(v) for v in data['values']], deserialize_type(data['upper_bound']), data['variance'], From d3ae7b1679e913f9fc22e0d8a23cc93531170fbc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 14 Apr 2022 17:40:25 +0100 Subject: [PATCH 6/8] Clean up tests --- test-data/unit/check-selftype.test | 60 +----------------------------- 1 file changed, 1 insertion(+), 59 deletions(-) diff --git a/test-data/unit/check-selftype.test b/test-data/unit/check-selftype.test index 61c101fa03ae..085c522c3013 100644 --- a/test-data/unit/check-selftype.test +++ b/test-data/unit/check-selftype.test @@ -1171,7 +1171,7 @@ def build_sub_wrapper_non_gen(descriptor: Descriptor[int]) -> SubWrapper[str]: return SubWrapper.create_wrapper(descriptor) # E: Argument 1 to "create_wrapper" of "BaseWrapper" has incompatible type "Descriptor[int]"; expected "Descriptor[str]" [builtins fixtures/classmethod.pyi] -[case testSelfTypeXXX] +[case testSelfTypeInGenericClassUsedFromAnotherGenericClass1] from typing import TypeVar, Generic, Iterator, List, Tuple _T_co = TypeVar("_T_co", covariant=True) @@ -1196,61 +1196,3 @@ class C(Generic[T]): for x, y in Z(self.a, self.b): reveal_type((x, y)) # N: Revealed type is "Tuple[T`1, builtins.str]" [builtins fixtures/tuple.pyi] - -[case testSelfTypeYYY] -from typing import TypeVar, Generic, Iterator, List, Tuple - -_T_co = TypeVar("_T_co", covariant=True) -_T1 = TypeVar("_T1") -_T2 = TypeVar("_T2") -S = TypeVar("S") - -class Z(Iterator[_T_co]): - def __new__(cls, - __iter1: List[_T1], - __iter2: List[_T2]) -> Z[Tuple[_T1, _T2]]: ... - def __iter__(self) -> Z[_T_co]: ... - def __next__(self) -> _T_co: ... - -T = TypeVar('T') - -class C(Generic[T]): - a: List[T] - b: List[str] - - def f(self) -> None: - for x, y in Z(self.a, self.b): - reveal_type((x, y)) # N: Revealed type is "Tuple[T`1, builtins.str]" -[builtins fixtures/tuple.pyi] - -[case testSelfTypeZZZ] -from typing import TypeVar, Generic, Tuple - -_T_co = TypeVar("_T_co", covariant=True) -S = TypeVar("S") - -class Z(Generic[_T_co]): - def m(self: S) -> S: ... - -T = TypeVar('T') - -class C(Generic[T]): - def f(self, z: Z[Tuple[T, str]]) -> None: - reveal_type(z.m()) # N: Revealed type is "__main__.Z[Tuple[T`1, builtins.str]]" -[builtins fixtures/tuple.pyi] - -[case testSelfTypeCCC] -from typing import Generic, TypeVar, Tuple - -T = TypeVar('T') -S = TypeVar('S') -U = TypeVar('U') -V = TypeVar('V') - -class C(Generic[T]): - def magic(self: C[Tuple[S, U]]) -> Tuple[T, S, U]: ... - -class D(Generic[V]): - def f(self) -> None: - reveal_type(C[Tuple[V, str]]().magic()) # N: Revealed type is "Tuple[Tuple[V`1, builtins.str], V`1, builtins.str]" -[builtins fixtures/tuple.pyi] From b38953a23362cb74fb13ac2461e8b9304aa1bf3e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 14 Apr 2022 17:47:19 +0100 Subject: [PATCH 7/8] Update comment --- mypy/types.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index bcf848c477c2..7bb978d4ce0e 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -416,8 +416,9 @@ class TypeVarId: # Class variable used for allocating fresh ids for metavariables. next_raw_id: ClassVar[int] = 1 - # Fullname of class or function which declares this type variable - # (not the fullname of the TypeVar definition!), or '' + # Fullname of class (or potentially function in the future) which + # declares this type variable (not the fullname of the TypeVar + # definition!), or '' namespace: str def __init__(self, raw_id: int, meta_level: int = 0, *, namespace: str = '') -> None: From 70b4ab2724aa0e1861f410fdfdda08cea7799275 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 14 Apr 2022 17:48:33 +0100 Subject: [PATCH 8/8] Add TODO --- mypy/tvar_scope.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index 14d10714942b..ac82a7708f0e 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -65,6 +65,7 @@ def bind_new(self, name: str, tvar_expr: TypeVarLikeExpr) -> TypeVarLikeType: else: self.func_id -= 1 i = self.func_id + # TODO: Consider also using namespaces for functions namespace = '' if isinstance(tvar_expr, TypeVarExpr): tvar_def: TypeVarLikeType = TypeVarType(