From 23b441efd78513d2c2ec5827574fee9f4d1aeb40 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 21 Nov 2018 10:46:56 -0800 Subject: [PATCH 1/2] Add LiteralType and stub methods to type visitors This diff adds a 'LiteralType' class to types.py and adds a corresponding stub method to all of the type visitors. Most of these stub methods just throw a 'NotImplementedError', though I did pencil in a few implementations that looked obvious. The reason I'm doing this now instead of later is because I want to start by getting these kind of big, gross, multi-file changes out of the way early and try and have any subsequent pull requests be more focused. I also want to formally confirm that the correct approach here *is* to create a new 'LiteralType' class before I start on the rest of the implementation work. (Other possible approaches would be to tack on some extra attribute to 'Instance', or make 'LiteralType' a subclass of 'Instance'.) No tests, sorry. My plan is to work on modifying the parsing and semantic analysis steps next before returning back to these the unimplemented methods and add tests then. I'll be opening a tracking issue momentarily to keep track of these TODOs. Tests shouldn't be necessary, in any case: everything I added just now is basically dead code. --- mypy/constraints.py | 5 +++- mypy/erasetype.py | 6 ++++- mypy/expandtype.py | 6 ++++- mypy/fixup.py | 5 +++- mypy/indirection.py | 3 +++ mypy/join.py | 7 ++++-- mypy/meet.py | 5 +++- mypy/sametypes.py | 10 +++++++- mypy/server/astdiff.py | 5 +++- mypy/server/astmerge.py | 5 +++- mypy/server/deps.py | 5 +++- mypy/subtypes.py | 8 ++++++- mypy/type_visitor.py | 19 ++++++++++++++- mypy/typeanal.py | 9 ++++++- mypy/types.py | 53 +++++++++++++++++++++++++++++++++++++++++ 15 files changed, 137 insertions(+), 14 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 5ca0c2670cc4..ea817765edda 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -5,7 +5,7 @@ from mypy.types import ( CallableType, Type, TypeVisitor, UnboundType, AnyType, NoneTyp, TypeVarType, Instance, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType, DeletedType, - UninhabitedType, TypeType, TypeVarId, TypeQuery, is_named_instance, TypeOfAny + UninhabitedType, TypeType, TypeVarId, TypeQuery, is_named_instance, TypeOfAny, LiteralType, ) from mypy.maptype import map_instance_to_supertype from mypy import nodes @@ -506,6 +506,9 @@ def visit_typeddict_type(self, template: TypedDictType) -> List[Constraint]: else: return [] + def visit_literal_type(self, template: LiteralType) -> List[Constraint]: + raise NotImplementedError() + def visit_union_type(self, template: UnionType) -> List[Constraint]: assert False, ("Unexpected UnionType in ConstraintBuilderVisitor" " (should have been handled in infer_constraints)") diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 1006092376bc..096fb0084c62 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -3,7 +3,7 @@ from mypy.types import ( Type, TypeVisitor, UnboundType, AnyType, NoneTyp, TypeVarId, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType, - DeletedType, TypeTranslator, UninhabitedType, TypeType, TypeOfAny + DeletedType, TypeTranslator, UninhabitedType, TypeType, TypeOfAny, LiteralType, ) from mypy.nodes import ARG_STAR, ARG_STAR2 @@ -78,6 +78,10 @@ def visit_tuple_type(self, t: TupleType) -> Type: def visit_typeddict_type(self, t: TypedDictType) -> Type: return t.fallback.accept(self) + def visit_literal_type(self, t: LiteralType) -> Type: + # TODO: Verify this implementation is correct + return t.fallback.accept(self) + def visit_union_type(self, t: UnionType) -> Type: erased_items = [erase_type(item) for item in t.items] return UnionType.make_simplified_union(erased_items) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index d4cd0d12d165..bf51dc2aa9a9 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -4,7 +4,7 @@ Type, Instance, CallableType, TypeVisitor, UnboundType, AnyType, NoneTyp, TypeVarType, Overloaded, TupleType, TypedDictType, UnionType, ErasedType, PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId, - FunctionLike, TypeVarDef + FunctionLike, TypeVarDef, LiteralType, ) @@ -111,6 +111,10 @@ def visit_tuple_type(self, t: TupleType) -> Type: def visit_typeddict_type(self, t: TypedDictType) -> Type: return t.copy_modified(item_types=self.expand_types(t.items.values())) + def visit_literal_type(self, t: LiteralType) -> Type: + # TODO: Verify this implementation is correct + return t + def visit_union_type(self, t: UnionType) -> Type: # After substituting for type variables in t.items, # some of the resulting types might be subtypes of others. diff --git a/mypy/fixup.py b/mypy/fixup.py index 947f3bbb7afc..074a6193ec1f 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -9,7 +9,7 @@ ) from mypy.types import ( CallableType, Instance, Overloaded, TupleType, TypedDictType, - TypeVarType, UnboundType, UnionType, TypeVisitor, + TypeVarType, UnboundType, UnionType, TypeVisitor, LiteralType, TypeType, NOT_READY ) from mypy.visitor import NodeVisitor @@ -206,6 +206,9 @@ def visit_typeddict_type(self, tdt: TypedDictType) -> None: if tdt.fallback is not None: tdt.fallback.accept(self) + def visit_literal_type(self, lt: LiteralType) -> None: + lt.fallback.accept(self) + def visit_type_var(self, tvt: TypeVarType) -> None: if tvt.values: for vt in tvt.values: diff --git a/mypy/indirection.py b/mypy/indirection.py index 5e5c827def3c..2776277acaa7 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -90,6 +90,9 @@ def visit_tuple_type(self, t: types.TupleType) -> Set[str]: def visit_typeddict_type(self, t: types.TypedDictType) -> Set[str]: return self._visit(t.items.values()) | self._visit(t.fallback) + def visit_literal_type(self, t: types.LiteralType) -> Set[str]: + return self._visit(t.fallback) + def visit_star_type(self, t: types.StarType) -> Set[str]: return set() diff --git a/mypy/join.py b/mypy/join.py index 31d14d51aa04..faf5860bd86e 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -5,8 +5,8 @@ from mypy.types import ( Type, AnyType, NoneTyp, TypeVisitor, Instance, UnboundType, TypeVarType, CallableType, - TupleType, TypedDictType, ErasedType, UnionType, FunctionLike, Overloaded, - PartialType, DeletedType, UninhabitedType, TypeType, true_or_false, TypeOfAny + TupleType, TypedDictType, ErasedType, UnionType, FunctionLike, Overloaded, LiteralType, + PartialType, DeletedType, UninhabitedType, TypeType, true_or_false, TypeOfAny, ) from mypy.maptype import map_instance_to_supertype from mypy.subtypes import ( @@ -267,6 +267,9 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: else: return self.default(self.s) + def visit_literal_type(self, t: LiteralType) -> Type: + raise NotImplementedError() + def visit_partial_type(self, t: PartialType) -> Type: # We only have partial information so we can't decide the join result. We should # never get here. diff --git a/mypy/meet.py b/mypy/meet.py index c7b46c11aff2..af2a8bc38ad9 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -7,7 +7,7 @@ from mypy.types import ( Type, AnyType, TypeVisitor, UnboundType, NoneTyp, TypeVarType, Instance, CallableType, TupleType, TypedDictType, ErasedType, UnionType, PartialType, DeletedType, - UninhabitedType, TypeType, TypeOfAny, Overloaded, FunctionLike, + UninhabitedType, TypeType, TypeOfAny, Overloaded, FunctionLike, LiteralType, ) from mypy.subtypes import ( is_equivalent, is_subtype, is_protocol_implementation, is_callable_compatible, @@ -520,6 +520,9 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: else: return self.default(self.s) + def visit_literal_type(self, t: LiteralType) -> Type: + raise NotImplementedError() + def visit_partial_type(self, t: PartialType) -> Type: # We can't determine the meet of partial types. We should never get here. assert False, 'Internal error' diff --git a/mypy/sametypes.py b/mypy/sametypes.py index ef053a5b4b19..1cb826a5ec4f 100644 --- a/mypy/sametypes.py +++ b/mypy/sametypes.py @@ -3,7 +3,7 @@ from mypy.types import ( Type, UnboundType, AnyType, NoneTyp, TupleType, TypedDictType, UnionType, CallableType, TypeVarType, Instance, TypeVisitor, ErasedType, - Overloaded, PartialType, DeletedType, UninhabitedType, TypeType + Overloaded, PartialType, DeletedType, UninhabitedType, TypeType, LiteralType, ) @@ -114,6 +114,14 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: else: return False + def visit_literal_type(self, left: LiteralType) -> bool: + if isinstance(self.right, LiteralType): + if left.value != self.right.value: + return False + return is_same_type(left.fallback, self.right.fallback) + else: + return False + def visit_union_type(self, left: UnionType) -> bool: if isinstance(self.right, UnionType): # Check that everything in left is in right diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 2aa2618c5043..8697358a4205 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -59,7 +59,7 @@ class level -- these are handled at attribute level (say, 'mod.Cls.method' from mypy.types import ( Type, TypeVisitor, UnboundType, AnyType, NoneTyp, UninhabitedType, ErasedType, DeletedType, Instance, TypeVarType, CallableType, TupleType, TypedDictType, - UnionType, Overloaded, PartialType, TypeType + UnionType, Overloaded, PartialType, TypeType, LiteralType, ) from mypy.util import get_prefix @@ -315,6 +315,9 @@ def visit_typeddict_type(self, typ: TypedDictType) -> SnapshotItem: required = tuple(sorted(typ.required_keys)) return ('TypedDictType', items, required) + def visit_literal_type(self, typ: LiteralType) -> SnapshotItem: + return ('LiteralType', typ.value, snapshot_type(typ.fallback)) + def visit_union_type(self, typ: UnionType) -> SnapshotItem: # Sort and remove duplicates so that we can use equality to test for # equivalent union type snapshots. diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index a7726d027c0c..241cdc988112 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -58,7 +58,7 @@ from mypy.types import ( Type, SyntheticTypeVisitor, Instance, AnyType, NoneTyp, CallableType, DeletedType, PartialType, TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType, - Overloaded, TypeVarDef, TypeList, CallableArgument, EllipsisType, StarType + Overloaded, TypeVarDef, TypeList, CallableArgument, EllipsisType, StarType, LiteralType, ) from mypy.util import get_prefix, replace_object_state from mypy.typestate import TypeState @@ -391,6 +391,9 @@ def visit_typeddict_type(self, typ: TypedDictType) -> None: value_type.accept(self) typ.fallback.accept(self) + def visit_literal_type(self, typ: LiteralType) -> None: + typ.fallback.accept(self) + def visit_unbound_type(self, typ: UnboundType) -> None: for arg in typ.args: arg.accept(self) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index f24b0a314a61..e0580ae3ef83 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -99,7 +99,7 @@ class 'mod.Cls'. This can also refer to an attribute inherited from a from mypy.types import ( Type, Instance, AnyType, NoneTyp, TypeVisitor, CallableType, DeletedType, PartialType, TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType, - FunctionLike, ForwardRef, Overloaded, TypeOfAny + FunctionLike, ForwardRef, Overloaded, TypeOfAny, LiteralType, ) from mypy.server.trigger import make_trigger, make_wildcard_trigger from mypy.util import correct_relative_import @@ -949,6 +949,9 @@ def visit_typeddict_type(self, typ: TypedDictType) -> List[str]: triggers.extend(self.get_type_triggers(typ.fallback)) return triggers + def visit_literal_type(self, typ: LiteralType) -> List[str]: + return self.get_type_triggers(typ.fallback) + def visit_unbound_type(self, typ: UnboundType) -> List[str]: return [] diff --git a/mypy/subtypes.py b/mypy/subtypes.py index d06ad40fdfa1..0d8e62fa6d48 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -5,7 +5,7 @@ Type, AnyType, UnboundType, TypeVisitor, FormalArgument, NoneTyp, function_type, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance, - FunctionLike, TypeOfAny + FunctionLike, TypeOfAny, LiteralType, ) import mypy.applytype import mypy.constraints @@ -327,6 +327,9 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: else: return False + def visit_literal_type(self, t: LiteralType) -> bool: + raise NotImplementedError() + def visit_overloaded(self, left: Overloaded) -> bool: right = self.right if isinstance(right, Instance): @@ -1168,6 +1171,9 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: return True return self._is_proper_subtype(left.fallback, right) + def visit_literal_type(self, left: LiteralType) -> bool: + raise NotImplementedError() + def visit_overloaded(self, left: Overloaded) -> bool: # TODO: What's the right thing to do here? return False diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index c882883c9d01..e61776b02cc5 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -19,7 +19,7 @@ T = TypeVar('T') from mypy.types import ( - Type, AnyType, CallableType, FunctionLike, Overloaded, TupleType, TypedDictType, + Type, AnyType, CallableType, FunctionLike, Overloaded, TupleType, TypedDictType, LiteralType, Instance, NoneTyp, TypeType, TypeOfAny, UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType, TypeVarDef, UnboundType, ErasedType, ForwardRef, StarType, EllipsisType, TypeList, CallableArgument, @@ -85,6 +85,10 @@ def visit_tuple_type(self, t: TupleType) -> T: def visit_typeddict_type(self, t: TypedDictType) -> T: pass + @abstractmethod + def visit_literal_type(self, t: LiteralType) -> T: + pass + @abstractmethod def visit_union_type(self, t: UnionType) -> T: pass @@ -181,6 +185,16 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: cast(Any, t.fallback.accept(self)), t.line, t.column) + def visit_literal_type(self, t: LiteralType) -> Type: + fallback = t.fallback.accept(self) + assert isinstance(fallback, Instance) + return LiteralType( + value=t.value, + fallback=fallback, + line=t.line, + column=t.column, + ) + def visit_union_type(self, t: UnionType) -> Type: return UnionType(self.translate_types(t.items), t.line, t.column) @@ -264,6 +278,9 @@ def visit_tuple_type(self, t: TupleType) -> T: def visit_typeddict_type(self, t: TypedDictType) -> T: return self.query_types(t.items.values()) + def visit_literal_type(self, t: LiteralType) -> T: + return self.strategy([]) + def visit_star_type(self, t: StarType) -> T: return t.type.accept(self) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 802287d54860..b7a7f2dea98b 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -15,7 +15,8 @@ Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, AnyType, CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, SyntheticTypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, - CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny, ForwardRef, Overloaded + CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny, ForwardRef, Overloaded, + LiteralType, ) from mypy.nodes import ( @@ -459,6 +460,9 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: ]) return TypedDictType(items, set(t.required_keys), t.fallback) + def visit_literal_type(self, t: LiteralType) -> Type: + raise NotImplementedError() + def visit_star_type(self, t: StarType) -> Type: return StarType(self.anal_type(t.type), t.line) @@ -754,6 +758,9 @@ def visit_typeddict_type(self, t: TypedDictType) -> None: for item_type in t.items.values(): item_type.accept(self) + def visit_literal_type(self, t: LiteralType) -> None: + raise NotImplementedError() + def visit_union_type(self, t: UnionType) -> None: for item in t.items: item.accept(self) diff --git a/mypy/types.py b/mypy/types.py index ecfdeaaccc97..2c39b18d0c30 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -27,6 +27,19 @@ JsonDict = Dict[str, Any] +# The set of all valid expressions that can currently be contained +# inside of a Literal[...]. +# +# Literals can contain enum-values: we special-case those and +# store the value as a string. +# +# Note: this type also happens to correspond to types that can be +# directly converted into JSON. The serialize/deserialize methods +# of 'LiteralType' rely on this, as well 'server.astdiff.SnapshotTypeVisitor' +# and 'types.TypeStrVisitor'. If we end up adding any non-JSON-serializable +# types to this list, we should make sure to edit those methods to match. +LiteralInnerExpr = Union[int, str, bool, None] + # If we only import type_visitor in the middle of the file, mypy # breaks, and if we do it at the top, it breaks at runtime because of # import cycle issues, so we do it at the top while typechecking and @@ -1240,6 +1253,43 @@ def zipall(self, right: 'TypedDictType') \ yield (item_name, None, right_item_type) +class LiteralType(Type): + __slots__ = ('value', 'fallback') + + def __init__(self, value: LiteralInnerExpr, fallback: Instance, + line: int = -1, column: int = -1) -> None: + super().__init__(line, column) + self.value = value + self.fallback = fallback + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_literal_type(self) + + def __hash__(self) -> int: + return hash((self.value, self.fallback)) + + def __eq__(self, other: object) -> bool: + if isinstance(other, LiteralType): + return self.fallback == other.fallback and self.value == other.value + else: + return NotImplemented + + def serialize(self) -> Union[JsonDict, str]: + return { + '.class': 'LiteralType', + 'value': self.value, + 'fallback': self.fallback.serialize(), + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'LiteralType': + assert data['.class'] == 'LiteralType' + return LiteralType( + value=data['value'], + fallback=Instance.deserialize(data['fallback']), + ) + + class StarType(Type): """The star type *type_parameter. @@ -1693,6 +1743,9 @@ def item_str(name: str, typ: str) -> str: suffix = ', fallback={}'.format(t.fallback.accept(self)) return 'TypedDict({}{}{})'.format(prefix, s, suffix) + def visit_literal_type(self, t: LiteralType) -> str: + return 'Literal[{}]'.format(t.value) + def visit_star_type(self, t: StarType) -> str: s = t.type.accept(self) return '*{}'.format(s) From 4da5f3c1802f70e3ad007823fabde38c7d30df78 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 22 Nov 2018 10:24:58 -0800 Subject: [PATCH 2/2] Respond to code review --- mypy/erasetype.py | 6 ++++-- mypy/types.py | 20 +++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 096fb0084c62..fa3e4abf79b6 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -79,8 +79,10 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: return t.fallback.accept(self) def visit_literal_type(self, t: LiteralType) -> Type: - # TODO: Verify this implementation is correct - return t.fallback.accept(self) + # The fallback for literal types should always be either + # something like int or str, or an enum class -- types that + # don't contain any TypeVars. So there's no need to visit it. + return t def visit_union_type(self, t: UnionType) -> Type: erased_items = [erase_type(item) for item in t.items] diff --git a/mypy/types.py b/mypy/types.py index 2c39b18d0c30..cf9c887d6782 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -33,12 +33,15 @@ # Literals can contain enum-values: we special-case those and # store the value as a string. # +# TODO: confirm that we're happy with representing enums (and the +# other types) in the manner described above. +# # Note: this type also happens to correspond to types that can be # directly converted into JSON. The serialize/deserialize methods # of 'LiteralType' rely on this, as well 'server.astdiff.SnapshotTypeVisitor' # and 'types.TypeStrVisitor'. If we end up adding any non-JSON-serializable # types to this list, we should make sure to edit those methods to match. -LiteralInnerExpr = Union[int, str, bool, None] +LiteralValue = Union[int, str, bool, None] # If we only import type_visitor in the middle of the file, mypy # breaks, and if we do it at the top, it breaks at runtime because of @@ -1254,9 +1257,20 @@ def zipall(self, right: 'TypedDictType') \ class LiteralType(Type): + """The type of a Literal instance. Literal[Value] + + A Literal always consists of: + + 1. A native Python object corresponding to the contained inner value + 2. A fallback for this Literal. The fallback also corresponds to the + parent type this Literal subtypes. + + For example, 'Literal[42]' is represented as + 'LiteralType(value=42, fallback=instance_of_int)' + """ __slots__ = ('value', 'fallback') - def __init__(self, value: LiteralInnerExpr, fallback: Instance, + def __init__(self, value: LiteralValue, fallback: Instance, line: int = -1, column: int = -1) -> None: super().__init__(line, column) self.value = value @@ -1744,7 +1758,7 @@ def item_str(name: str, typ: str) -> str: return 'TypedDict({}{}{})'.format(prefix, s, suffix) def visit_literal_type(self, t: LiteralType) -> str: - return 'Literal[{}]'.format(t.value) + return 'Literal[{}]'.format(repr(t.value)) def visit_star_type(self, t: StarType) -> str: s = t.type.accept(self)