Support recursive NamedTuple classes

mypy/semanal.py

@@ -1214,22 +1214,22 @@ def analyze_class_body_common(self, defn: ClassDef) -> None:
 
     def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool:
         """Check if this class can define a named tuple."""
-        if defn.info and defn.info.is_named_tuple:
-            # Don't reprocess everything. We just need to process methods defined
-            # in the named tuple class body.
-            is_named_tuple, info = True, defn.info  # type: bool, Optional[TypeInfo]
-        else:
-            is_named_tuple, info = self.named_tuple_analyzer.analyze_namedtuple_classdef(
-                defn, self.is_stub_file, self.is_func_scope())
+        if self.named_tuple_analyzer.is_incomplete_namedtuple_classdef(defn):
+            self.prepare_class_def(defn)
+
+        is_named_tuple, complete = self.named_tuple_analyzer.analyze_namedtuple_classdef(
+            defn, self.is_stub_file, self.is_func_scope())
+
         if is_named_tuple:
-            if info is None:
+            if not complete:
                 self.mark_incomplete(defn.name, defn)
             else:
-                self.prepare_class_def(defn, info)
+                self.prepare_class_def(defn)
                 with self.scope.class_scope(defn.info):
-                    with self.named_tuple_analyzer.save_namedtuple_body(info):
+                    with self.named_tuple_analyzer.save_namedtuple_body(defn.info):
                         self.analyze_class_body_common(defn)
             return True
+
         return False
 
     def apply_class_plugin_hooks(self, defn: ClassDef) -> None:
@@ -1462,10 +1462,16 @@ def prepare_class_def(self, defn: ClassDef, info: Optional[TypeInfo] = None) ->
                 info._fullname = self.qualified_name(defn.name)
             else:
                 info._fullname = info.name
+
         local_name = defn.name
         if '@' in local_name:
             local_name = local_name.split('@')[0]
-        self.add_symbol(local_name, defn.info, defn)
+
+        # Add symbol, unless in func scope and intermediate completion of named tuple class
+        if not (self.is_nested_within_func_scope()
+                and self.named_tuple_analyzer.is_incomplete_namedtuple_classdef(defn)):
+            self.add_symbol(local_name, defn.info, defn)
+
         if self.is_nested_within_func_scope():
             # We need to preserve local classes, let's store them
             # in globals under mangled unique names

mypy/semanal_namedtuple.py

@@ -55,34 +55,40 @@ def __init__(self, options: Options, api: SemanticAnalyzerInterface) -> None:
 
     def analyze_namedtuple_classdef(self, defn: ClassDef, is_stub_file: bool,
                                     is_func_scope: bool
-                                    ) -> Tuple[bool, Optional[TypeInfo]]:
+                                    ) -> Tuple[bool, bool]:
         """Analyze if given class definition can be a named tuple definition.
 
         Return a tuple where first item indicates whether this can possibly be a named tuple,
-        and the second item is the corresponding TypeInfo (may be None if not ready and should be
-        deferred).
+        and the second item indicates whether definition is complete or requires another pass.
         """
         for base_expr in defn.base_type_exprs:
             if isinstance(base_expr, RefExpr):
                 self.api.accept(base_expr)
                 if base_expr.fullname == 'typing.NamedTuple':
-                    result = self.check_namedtuple_classdef(defn, is_stub_file)
-                    if result is None:
-                        # This is a valid named tuple, but some types are incomplete.
-                        return True, None
-                    items, types, default_items = result
-                    if is_func_scope and '@' not in defn.name:
-                        defn.name += '@' + str(defn.line)
-                    info = self.build_namedtuple_typeinfo(
-                        defn.name, items, types, default_items, defn.line)
-                    defn.info = info
-                    defn.analyzed = NamedTupleExpr(info, is_typed=True)
-                    defn.analyzed.line = defn.line
-                    defn.analyzed.column = defn.column
-                    # All done: this is a valid named tuple with all types known.
-                    return True, info
-        # This can't be a valid named tuple.
-        return False, None
+                    break
+        else:
+            # This can't be a valid named tuple.
+            return False, False
+
+        if not defn.info:
+            defn.info = self._basic_namedtuple_typeinfo(defn.name, defn.line)
+        result = self.check_namedtuple_classdef(defn, is_stub_file)
+        if result is None:
+            # This is a valid named tuple, but some types are incomplete.
+            return True, False
+
+        items, types, default_items = result
+        if is_func_scope and '@' not in defn.name:
+            defn.name += '@' + str(defn.line)
+        self._complete_namedtuple_typeinfo(defn.info, items, types, default_items)
+        defn.analyzed = NamedTupleExpr(defn.info, is_typed=True)
+        defn.analyzed.line = defn.line
+        defn.analyzed.column = defn.column
+        # All done: this is a valid named tuple with all types known.
+        return True, True
+
+    def is_incomplete_namedtuple_classdef(self, defn: ClassDef) -> bool:
+        return bool(defn.info) and defn.info.is_named_tuple and defn.info.tuple_type is None
 
     def check_namedtuple_classdef(self, defn: ClassDef, is_stub_file: bool
                                   ) -> Optional[Tuple[List[str],
@@ -392,28 +398,30 @@ def build_namedtuple_typeinfo(self,
                                   types: List[Type],
                                   default_items: Mapping[str, Expression],
                                   line: int) -> TypeInfo:
-        strtype = self.api.named_type('builtins.str')
+        info = self._basic_namedtuple_typeinfo(name, line)
+        self._complete_namedtuple_typeinfo(info, items, types, default_items)
+        return info
+
+    def _basic_namedtuple_typeinfo(self,
+                                  name: str,
+                                  line: int) -> TypeInfo:
         implicit_any = AnyType(TypeOfAny.special_form)
-        basetuple_type = self.api.named_type('builtins.tuple', [implicit_any])
-        dictype = (self.api.named_type_or_none('builtins.dict', [strtype, implicit_any])
-                   or self.api.named_type('builtins.object'))
-        # Actual signature should return OrderedDict[str, Union[types]]
-        ordereddictype = (self.api.named_type_or_none('builtins.dict', [strtype, implicit_any])
-                          or self.api.named_type('builtins.object'))
         fallback = self.api.named_type('builtins.tuple', [implicit_any])
-        # Note: actual signature should accept an invariant version of Iterable[UnionType[types]].
-        # but it can't be expressed. 'new' and 'len' should be callable types.
-        iterable_type = self.api.named_type_or_none('typing.Iterable', [implicit_any])
-        function_type = self.api.named_type('builtins.function')
-
-        literals: List[Type] = [LiteralType(item, strtype) for item in items]
-        match_args_type = TupleType(literals, basetuple_type)
 
         info = self.api.basic_new_typeinfo(name, fallback, line)
         info.is_named_tuple = True
-        tuple_base = TupleType(types, fallback)
-        info.tuple_type = tuple_base
         info.line = line
+
+        return info
+
+    def _complete_namedtuple_typeinfo(self,
+                                  info: TypeInfo,
+                                  items: List[str],
+                                  types: List[Type],
+                                  default_items: Mapping[str, Expression]):
+        tuple_base = TupleType(types, info.bases[0])
+        info.tuple_type = tuple_base
+
         # For use by mypyc.
         info.metadata['namedtuple'] = {'fields': items.copy()}
 
@@ -440,7 +448,23 @@ def add_field(var: Var, is_initialized_in_class: bool = False,
         # are analyzed).
         vars = [Var(item, typ) for item, typ in zip(items, types)]
 
+        strtype = self.api.named_type('builtins.str')
+        implicit_any = AnyType(TypeOfAny.special_form)
+        basetuple_type = self.api.named_type('builtins.tuple', [implicit_any])
+        dictype = (self.api.named_type_or_none('builtins.dict', [strtype, implicit_any])
+                   or self.api.named_type('builtins.object'))
+        # Actual signature should return OrderedDict[str, Union[types]]
+        ordereddictype = (self.api.named_type_or_none('builtins.dict', [strtype, implicit_any])
+                          or self.api.named_type('builtins.object'))
         tuple_of_strings = TupleType([strtype for _ in items], basetuple_type)
+        literals: List[Type] = [LiteralType(item, strtype) for item in items]
+        match_args_type = TupleType(literals, basetuple_type)
+
+        # Note: actual signature should accept an invariant version of Iterable[UnionType[types]].
+        # but it can't be expressed. 'new' and 'len' should be callable types.
+        iterable_type = self.api.named_type_or_none('typing.Iterable', [implicit_any])
+        function_type = self.api.named_type('builtins.function')
+
         add_field(Var('_fields', tuple_of_strings), is_initialized_in_class=True)
         add_field(Var('_field_types', dictype), is_initialized_in_class=True)
         add_field(Var('_field_defaults', dictype), is_initialized_in_class=True)
@@ -477,15 +501,15 @@ def add_method(funcname: str,
             func.is_class = is_classmethod
             func.type = set_callable_name(signature, func)
             func._fullname = info.fullname + '.' + funcname
-            func.line = line
+            func.line = info.line
             if is_classmethod:
                 v = Var(funcname, func.type)
                 v.is_classmethod = True
                 v.info = info
                 v._fullname = func._fullname
                 func.is_decorated = True
                 dec = Decorator(func, [NameExpr('classmethod')], v)
-                dec.line = line
+                dec.line = info.line
                 sym = SymbolTableNode(MDEF, dec)
             else:
                 sym = SymbolTableNode(MDEF, func)
@@ -513,7 +537,6 @@ def make_init_arg(var: Var) -> Argument:
         self_tvar_expr = TypeVarExpr(SELF_TVAR_NAME, info.fullname + '.' + SELF_TVAR_NAME,
                                      [], info.tuple_type)
         info.names[SELF_TVAR_NAME] = SymbolTableNode(MDEF, self_tvar_expr)
-        return info
 
     @contextmanager
     def save_namedtuple_body(self, named_tuple_info: TypeInfo) -> Iterator[None]:

test-data/unit/check-incremental.test

@@ -5677,6 +5677,7 @@ from typing_extensions import TypedDict, TypeAlias
 from enum import Enum
 from dataclasses import dataclass
 
+reveal_type(1) # TODO: why does this prevent an error?
 class C:
     def f(self) -> None:
         class C:
@@ -5709,7 +5710,10 @@ class C:
         n: N = N(NT1(c=1))
 
 [builtins fixtures/dict.pyi]
+[out1]
+tmp/b.py:6: note: Revealed type is "Literal[1]?"
 [out2]
+tmp/b.py:6: note: Revealed type is "Literal[1]?"
 tmp/a.py:2: error: "object" has no attribute "xyz"
 
 [case testIncrementalInvalidNamedTupleInUnannotatedFunction]

test-data/unit/check-namedtuple.test

@@ -630,13 +630,13 @@ tmp/b.py:7: note: Revealed type is "Tuple[Any, fallback=a.N]"
 
 from typing import NamedTuple
 class MyNamedTuple(NamedTuple):
-    parent: 'MyNamedTuple' # E: Cannot resolve name "MyNamedTuple" (possible cyclic definition)
+    parent: 'MyNamedTuple'
 
 def bar(nt: MyNamedTuple) -> MyNamedTuple:
     return nt
 
 x: MyNamedTuple
-reveal_type(x.parent) # N: Revealed type is "Any"
+reveal_type(x.parent) # N: Revealed type is "__main__.MyNamedTuple"
 [builtins fixtures/tuple.pyi]
 
 -- Some crazy self-referential named tuples and types dicts
@@ -682,22 +682,22 @@ from typing import Tuple, NamedTuple
 
 A = NamedTuple('A', [
         ('x', str),
-        ('y', Tuple['B', ...]), # E: Cannot resolve name "B" (possible cyclic definition)
+        ('y', Tuple['B', ...]),
     ])
 class B(NamedTuple):
     x: A
     y: int
 
 n: A
-reveal_type(n) # N: Revealed type is "Tuple[builtins.str, builtins.tuple[Any, ...], fallback=__main__.A]"
+reveal_type(n) # N: Revealed type is "Tuple[builtins.str, builtins.tuple[__main__.B, ...], fallback=__main__.A]"
 [builtins fixtures/tuple.pyi]
 
 [case testSelfRefNT3]
 
 from typing import NamedTuple, Tuple
 
 class B(NamedTuple):
-    x: Tuple[A, int] # E: Cannot resolve name "A" (possible cyclic definition)
+    x: Tuple[A, int]
     y: int
 
 A = NamedTuple('A', [
@@ -706,26 +706,26 @@ A = NamedTuple('A', [
     ])
 n: B
 m: A
-reveal_type(n.x) # N: Revealed type is "Tuple[Any, builtins.int]"
+reveal_type(n.x) # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.B, fallback=__main__.A], builtins.int]"
 reveal_type(m[0]) # N: Revealed type is "builtins.str"
 lst = [m, n]
-reveal_type(lst[0]) # N: Revealed type is "Tuple[builtins.object, builtins.object]"
+reveal_type(lst[0]) # N: Revealed type is "builtins.tuple[builtins.object, ...]"
 [builtins fixtures/tuple.pyi]
 
 [case testSelfRefNT4]
 
 from typing import NamedTuple
 
 class B(NamedTuple):
-    x: A # E: Cannot resolve name "A" (possible cyclic definition)
+    x: A
     y: int
 
 class A(NamedTuple):
     x: str
     y: B
 
 n: A
-reveal_type(n.y[0]) # N: Revealed type is "Any"
+reveal_type(n.y[0]) # N: Revealed type is "builtins.object"
 [builtins fixtures/tuple.pyi]
 
 [case testSelfRefNT5]
@@ -795,13 +795,13 @@ tp = NamedTuple('tp', [('x', int)])
 from typing import List, NamedTuple
 
 class Command(NamedTuple):
-    subcommands: List['Command'] # E: Cannot resolve name "Command" (possible cyclic definition)
+    subcommands: List['Command']
 
 class HelpCommand(Command):
     pass
 
 hc = HelpCommand(subcommands=[])
-reveal_type(hc)  # N: Revealed type is "Tuple[builtins.list[Any], fallback=__main__.HelpCommand]"
+reveal_type(hc)  # N: Revealed type is "Tuple[builtins.list[__main__.Command], fallback=__main__.HelpCommand]"
 [builtins fixtures/list.pyi]
 [out]
 

test-data/unit/check-newsemanal.test

@@ -879,7 +879,7 @@ class Out(NamedTuple):
     x: In
     y: Other
 
-reveal_type(o)  # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.Other, fallback=__main__.In], __main__.Other, fallback=__main__.Out]"
+reveal_type(o)  # N: Revealed type is "__main__.Out"
 reveal_type(o.x)  # N: Revealed type is "Tuple[builtins.str, __main__.Other, fallback=__main__.In]"
 reveal_type(o.y)  # N: Revealed type is "__main__.Other"
 reveal_type(o.x.t)  # N: Revealed type is "__main__.Other"
@@ -916,7 +916,7 @@ from typing import NamedTuple
 o: C.Out
 i: C.In
 
-reveal_type(o)  # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.C.Other, fallback=__main__.C.In], __main__.C.Other, fallback=__main__.C.Out]"
+reveal_type(o)  # N: Revealed type is "__main__.C.Out"
 reveal_type(o.x)  # N: Revealed type is "Tuple[builtins.str, __main__.C.Other, fallback=__main__.C.In]"
 reveal_type(o.y)  # N: Revealed type is "__main__.C.Other"
 reveal_type(o.x.t)  # N: Revealed type is "__main__.C.Other"
@@ -951,9 +951,9 @@ class C:
 from typing import NamedTuple
 
 c = C()
-reveal_type(c.o)  # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.Other@18, fallback=__main__.C.In@15], __main__.Other@18, fallback=__main__.C.Out@11]"
-reveal_type(c.o.x)  # N: Revealed type is "Tuple[builtins.str, __main__.Other@18, fallback=__main__.C.In@15]"
-reveal_type(c.o.method())  # N: Revealed type is "Tuple[builtins.str, __main__.Other@18, fallback=__main__.C.In@15]"
+reveal_type(c.o)  # N: Revealed type is "Tuple[Tuple[builtins.str, __main__.Other@18, fallback=__main__.In@15], __main__.Other@18, fallback=__main__.Out@11]"
+reveal_type(c.o.x)  # N: Revealed type is "Tuple[builtins.str, __main__.Other@18, fallback=__main__.In@15]"
+reveal_type(c.o.method())  # N: Revealed type is "Tuple[builtins.str, __main__.Other@18, fallback=__main__.In@15]"
 
 class C:
     def get_tuple(self) -> None: