Remove RawLiteralType synthetic type

This diff changes how we track raw literal types in the semantic
analysis phase. It makes the following changes:

1.  Removes the `RawLiteralType` synthetic type.

2.  Adds a new `TypeOfAny`: `TypeOfAny.invalid_type` as suggested
    in python#4030.

3.  Modifies `AnyType` so it can optionally contain a new
    `RawLiteral` class. This class contains information
    about the underlying literal that produced that particular
    `TypeOfAny`.

4.  Adjusts mypy to stop recommending using `Literal[...]` when
    doing `A = NewType('A', 4)` or `T = TypeVar('T', bound=4)`.

    (The former suggestion is a bad one: you can't create a NewType of a
    Literal[...] type. The latter suggestion is a valid but stupid one:
    `T = TypeVar('T', bound=Literal[4])` is basically the same thing as
    `T = Literal[4]`.)

    This resolves python#5989.

The net effect of this diff is that:

1.  RawLiteralTypes no longer leak during fine-grained mode,
    which should partially help unblock
    python#6075.

2.  The way mypy handles literal expressions in types is "inverted".

    Previously, we by default assumed literal expressions would belong
    inside `Literal[...]` and tacked on some logic to make them convert
    into error `AnyTypes`. Now, we do the reverse: we start with an
    error `AnyType` and convert those into `Literal[...]`s as needed.

    This more closely mirrors the way mypy *used* to work before we
    started work on Literal types. It should also hopefully help reduce
    some of the cognitive burden of working on other parts of the
    semantic analysis code, since we no longer need to worry about the
    `RawLiteralType` synthetic type.

3.  We now have more flexibility in how we choose to handle invalid
    types: since they're just `Anys`, we have more opportunities to
    intercept and customize the exact way in which we handle errors.

    Also see python#4030 for additional
    context. (This diff lays out some of the foundation work for that
    diff).

mypy/exprtotype.py

@@ -8,7 +8,7 @@
 from mypy.fastparse import parse_type_string
 from mypy.types import (
     Type, UnboundType, TypeList, EllipsisType, AnyType, Optional, CallableArgument, TypeOfAny,
-    RawLiteralType,
+    RawLiteral,
 )
 
 
@@ -39,9 +39,19 @@ def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = No
     if isinstance(expr, NameExpr):
         name = expr.name
         if name == 'True':
-            return RawLiteralType(True, 'builtins.bool', line=expr.line, column=expr.column)
+            return AnyType(
+                TypeOfAny.invalid_type,
+                raw_literal=RawLiteral(True, 'builtins.bool'),
+                line=expr.line,
+                column=expr.column,
+            )
         elif name == 'False':
-            return RawLiteralType(False, 'builtins.bool', line=expr.line, column=expr.column)
+            return AnyType(
+                TypeOfAny.invalid_type,
+                raw_literal=RawLiteral(False, 'builtins.bool'),
+                line=expr.line,
+                column=expr.column,
+            )
         else:
             return UnboundType(name, line=expr.line, column=expr.column)
     elif isinstance(expr, MemberExpr):
@@ -122,17 +132,27 @@ def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = No
                                  assume_str_is_unicode=True)
     elif isinstance(expr, UnaryExpr):
         typ = expr_to_unanalyzed_type(expr.expr)
-        if isinstance(typ, RawLiteralType) and isinstance(typ.value, int) and expr.op == '-':
-            typ.value *= -1
-            return typ
-        else:
-            raise TypeTranslationError()
+        if isinstance(typ, AnyType) and typ.raw_literal is not None:
+            if isinstance(typ.raw_literal.value, int) and expr.op == '-':
+                typ.raw_literal.value *= -1
+                return typ
+        raise TypeTranslationError()
     elif isinstance(expr, IntExpr):
-        return RawLiteralType(expr.value, 'builtins.int', line=expr.line, column=expr.column)
+        return AnyType(
+            TypeOfAny.invalid_type,
+            raw_literal=RawLiteral(expr.value, 'builtins.int'),
+            line=expr.line,
+            column=expr.column,
+        )
     elif isinstance(expr, FloatExpr):
         # Floats are not valid parameters for RawLiteralType, so we just
         # pass in 'None' for now. We'll report the appropriate error at a later stage.
-        return RawLiteralType(None, 'builtins.float', line=expr.line, column=expr.column)
+        return AnyType(
+            TypeOfAny.invalid_type,
+            raw_literal=RawLiteral(None, 'builtins.float'),
+            line=expr.line,
+            column=expr.column,
+        )
     elif isinstance(expr, EllipsisExpr):
         return EllipsisType(expr.line)
     else:

mypy/fastparse.py

@@ -31,7 +31,7 @@
 )
 from mypy.types import (
     Type, CallableType, AnyType, UnboundType, TupleType, TypeList, EllipsisType, CallableArgument,
-    TypeOfAny, Instance, RawLiteralType,
+    TypeOfAny, Instance, RawLiteral,
 )
 from mypy import defaults
 from mypy import messages
@@ -184,11 +184,21 @@ def parse_type_string(expr_string: str, expr_fallback_name: str,
             node.original_str_fallback = expr_fallback_name
             return node
         else:
-            return RawLiteralType(expr_string, expr_fallback_name, line, column)
+            return AnyType(
+                TypeOfAny.invalid_type,
+                raw_literal=RawLiteral(expr_string, expr_fallback_name),
+                line=line,
+                column=column,
+            )
     except (SyntaxError, ValueError):
         # Note: the parser will raise a `ValueError` instead of a SyntaxError if
         # the string happens to contain things like \x00.
-        return RawLiteralType(expr_string, expr_fallback_name, line, column)
+        return AnyType(
+            TypeOfAny.invalid_type,
+            raw_literal=RawLiteral(expr_string, expr_fallback_name),
+            line=line,
+            column=column,
+        )
 
 
 def is_no_type_check_decorator(expr: ast3.expr) -> bool:
@@ -1183,7 +1193,11 @@ def visit_Name(self, n: Name) -> Type:
 
     def visit_NameConstant(self, n: NameConstant) -> Type:
         if isinstance(n.value, bool):
-            return RawLiteralType(n.value, 'builtins.bool', line=self.line)
+            return AnyType(
+                TypeOfAny.invalid_type,
+                raw_literal=RawLiteral(n.value, 'builtins.bool'),
+                line=self.line,
+            )
         else:
             return UnboundType(str(n.value), line=self.line)
 
@@ -1192,9 +1206,9 @@ def visit_UnaryOp(self, n: UnaryOp) -> Type:
         # We support specifically Literal[-4] and nothing else.
         # For example, Literal[+4] or Literal[~6] is not supported.
         typ = self.visit(n.operand)
-        if isinstance(typ, RawLiteralType) and isinstance(n.op, USub):
-            if isinstance(typ.value, int):
-                typ.value *= -1
+        if isinstance(typ, AnyType) and typ.raw_literal is not None:
+            if isinstance(typ.raw_literal.value, int) and isinstance(n.op, USub):
+                typ.raw_literal.value *= -1
                 return typ
         self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(n, 'col_offset', -1))
         return AnyType(TypeOfAny.from_error)
@@ -1204,11 +1218,19 @@ def visit_Num(self, n: Num) -> Type:
         # Could be either float or int
         numeric_value = n.n
         if isinstance(numeric_value, int):
-            return RawLiteralType(numeric_value, 'builtins.int', line=self.line)
+            return AnyType(
+                TypeOfAny.invalid_type,
+                raw_literal=RawLiteral(numeric_value, 'builtins.int'),
+                line=self.line,
+            )
         elif isinstance(numeric_value, float):
             # Floats and other numbers are not valid parameters for RawLiteralType, so we just
             # pass in 'None' for now. We'll report the appropriate error at a later stage.
-            return RawLiteralType(None, 'builtins.float', line=self.line)
+            return AnyType(
+                TypeOfAny.invalid_type,
+                raw_literal=RawLiteral(None, 'builtins.float'),
+                line=self.line,
+            )
         else:
             self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(n, 'col_offset', -1))
             return AnyType(TypeOfAny.from_error)
@@ -1230,7 +1252,12 @@ def visit_Str(self, n: Str) -> Type:
     # Bytes(bytes s)
     def visit_Bytes(self, n: Bytes) -> Type:
         contents = bytes_to_human_readable_repr(n.s)
-        return RawLiteralType(contents, 'builtins.bytes', self.line, column=n.col_offset)
+        return AnyType(
+            TypeOfAny.invalid_type,
+            raw_literal=RawLiteral(contents, 'builtins.bytes'),
+            line=self.line,
+            column=n.col_offset,
+        )
 
     # Subscript(expr value, slice slice, expr_context ctx)
     def visit_Subscript(self, n: ast3.Subscript) -> Type:

mypy/indirection.py

@@ -90,9 +90,6 @@ def visit_tuple_type(self, t: types.TupleType) -> Set[str]:
     def visit_typeddict_type(self, t: types.TypedDictType) -> Set[str]:
         return self._visit(t.items.values()) | self._visit(t.fallback)
 
-    def visit_raw_literal_type(self, t: types.RawLiteralType) -> Set[str]:
-        assert False, "Unexpected RawLiteralType after semantic analysis phase"
-
     def visit_literal_type(self, t: types.LiteralType) -> Set[str]:
         return self._visit(t.fallback)
 

mypy/plugin.py

@@ -180,6 +180,7 @@ def anal_type(self, t: Type, *,
                   tvar_scope: Optional[TypeVarScope] = None,
                   allow_tuple_literal: bool = False,
                   allow_unbound_tvars: bool = False,
+                  report_invalid_types: bool = True,
                   third_pass: bool = False) -> Type:
         """Analyze an unbound type."""
         raise NotImplementedError

mypy/semanal.py

@@ -1283,7 +1283,7 @@ def update_metaclass(self, defn: ClassDef) -> None:
             return
         defn.metaclass = metas.pop()
 
-    def expr_to_analyzed_type(self, expr: Expression) -> Type:
+    def expr_to_analyzed_type(self, expr: Expression, report_invalid_types: bool = True) -> Type:
         if isinstance(expr, CallExpr):
             expr.accept(self)
             info = self.named_tuple_analyzer.check_namedtuple(expr, None, self.is_func_scope())
@@ -1295,7 +1295,7 @@ def expr_to_analyzed_type(self, expr: Expression) -> Type:
             fallback = Instance(info, [])
             return TupleType(info.tuple_type.items, fallback=fallback)
         typ = expr_to_unanalyzed_type(expr)
-        return self.anal_type(typ)
+        return self.anal_type(typ, report_invalid_types=report_invalid_types)
 
     def verify_base_classes(self, defn: ClassDef) -> bool:
         info = defn.info
@@ -1686,6 +1686,7 @@ def type_analyzer(self, *,
                       tvar_scope: Optional[TypeVarScope] = None,
                       allow_tuple_literal: bool = False,
                       allow_unbound_tvars: bool = False,
+                      report_invalid_types: bool = True,
                       third_pass: bool = False) -> TypeAnalyser:
         if tvar_scope is None:
             tvar_scope = self.tvar_scope
@@ -1696,6 +1697,7 @@ def type_analyzer(self, *,
                             self.is_typeshed_stub_file,
                             allow_unbound_tvars=allow_unbound_tvars,
                             allow_tuple_literal=allow_tuple_literal,
+                            report_invalid_types=report_invalid_types,
                             allow_unnormalized=self.is_stub_file,
                             third_pass=third_pass)
         tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic())
@@ -1706,10 +1708,12 @@ def anal_type(self, t: Type, *,
                   tvar_scope: Optional[TypeVarScope] = None,
                   allow_tuple_literal: bool = False,
                   allow_unbound_tvars: bool = False,
+                  report_invalid_types: bool = True,
                   third_pass: bool = False) -> Type:
         a = self.type_analyzer(tvar_scope=tvar_scope,
                                allow_unbound_tvars=allow_unbound_tvars,
                                allow_tuple_literal=allow_tuple_literal,
+                               report_invalid_types=report_invalid_types,
                                third_pass=third_pass)
         typ = t.accept(a)
         self.add_type_alias_deps(a.aliases_used)
@@ -2394,7 +2398,12 @@ def process_typevar_parameters(self, args: List[Expression],
                     self.fail("TypeVar cannot have both values and an upper bound", context)
                     return None
                 try:
-                    upper_bound = self.expr_to_analyzed_type(param_value)
+                    upper_bound = self.expr_to_analyzed_type(param_value,
+                                                             report_invalid_types=False)
+                    if isinstance(upper_bound, AnyType) and upper_bound.from_invalid_type:
+                        self.fail("TypeVar 'bound' must be a type", param_value)
+                        # Note: we do not return 'None' here: we want to continue using the
+                        # AnyType as the upper bound.
                 except TypeTranslationError:
                     self.fail("TypeVar 'bound' must be a type", param_value)
                     return None

mypy/semanal_newtype.py

@@ -114,11 +114,12 @@ def check_newtype_args(self, name: str, call: CallExpr, context: Context) -> Opt
             self.fail(msg, context)
             return None
 
-        old_type = self.api.anal_type(unanalyzed_type)
+        old_type = self.api.anal_type(unanalyzed_type, report_invalid_types=False)
 
         # The caller of this function assumes that if we return a Type, it's always
         # a valid one. So, we translate AnyTypes created from errors into None.
-        if isinstance(old_type, AnyType) and old_type.type_of_any == TypeOfAny.from_error:
+        bad_anys = (TypeOfAny.from_error, TypeOfAny.invalid_type)
+        if isinstance(old_type, AnyType) and old_type.type_of_any in bad_anys:
             self.fail(msg, context)
             return None
 

mypy/semanal_shared.py

@@ -91,6 +91,7 @@ def anal_type(self, t: Type, *,
                   tvar_scope: Optional[TypeVarScope] = None,
                   allow_tuple_literal: bool = False,
                   allow_unbound_tvars: bool = False,
+                  report_invalid_types: bool = True,
                   third_pass: bool = False) -> Type:
         raise NotImplementedError
 

mypy/server/astdiff.py

@@ -267,7 +267,18 @@ def visit_unbound_type(self, typ: UnboundType) -> SnapshotItem:
                 snapshot_types(typ.args))
 
     def visit_any(self, typ: AnyType) -> SnapshotItem:
-        return snapshot_simple_type(typ)
+        if typ.raw_literal:
+            return (
+                'Any',
+                typ.type_of_any,
+                typ.raw_literal.value,
+                typ.raw_literal.base_type_name,
+            )
+        else:
+            return (
+                'Any',
+                typ.type_of_any,
+            )
 
     def visit_none_type(self, typ: NoneTyp) -> SnapshotItem:
         return snapshot_simple_type(typ)

mypy/server/astmerge.py

@@ -59,7 +59,6 @@
     Type, SyntheticTypeVisitor, Instance, AnyType, NoneTyp, CallableType, DeletedType, PartialType,
     TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType,
     Overloaded, TypeVarDef, TypeList, CallableArgument, EllipsisType, StarType, LiteralType,
-    RawLiteralType,
 )
 from mypy.util import get_prefix, replace_object_state
 from mypy.typestate import TypeState
@@ -392,9 +391,6 @@ def visit_typeddict_type(self, typ: TypedDictType) -> None:
             value_type.accept(self)
         typ.fallback.accept(self)
 
-    def visit_raw_literal_type(self, t: RawLiteralType) -> None:
-        assert False, "Unexpected RawLiteralType after semantic analysis phase"
-
     def visit_literal_type(self, typ: LiteralType) -> None:
         typ.fallback.accept(self)
 

mypy/type_visitor.py

@@ -20,7 +20,7 @@
 
 from mypy.types import (
     Type, AnyType, CallableType, Overloaded, TupleType, TypedDictType, LiteralType,
-    RawLiteralType, Instance, NoneTyp, TypeType,
+    Instance, NoneTyp, TypeType,
     UnionType, TypeVarType, PartialType, DeletedType, UninhabitedType, TypeVarDef,
     UnboundType, ErasedType, ForwardRef, StarType, EllipsisType, TypeList, CallableArgument,
 )
@@ -127,10 +127,6 @@ def visit_callable_argument(self, t: CallableArgument) -> T:
     def visit_ellipsis_type(self, t: EllipsisType) -> T:
         pass
 
-    @abstractmethod
-    def visit_raw_literal_type(self, t: RawLiteralType) -> T:
-        pass
-
 
 @trait
 class TypeTranslator(TypeVisitor[Type]):
@@ -282,9 +278,6 @@ def visit_tuple_type(self, t: TupleType) -> T:
     def visit_typeddict_type(self, t: TypedDictType) -> T:
         return self.query_types(t.items.values())
 
-    def visit_raw_literal_type(self, t: RawLiteralType) -> T:
-        return self.strategy([])
-
     def visit_literal_type(self, t: LiteralType) -> T:
         return self.strategy([])