From 20080f3b6757f039b199c3d23c630830ffc61cbc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 16 Nov 2018 16:45:53 +0000 Subject: [PATCH 01/17] Work towards fixing **kwargs that are typed dicts --- mypy/argmap.py | 34 +++++++++++++++++++---------- mypy/checkexpr.py | 16 +++++++++----- mypy/constraints.py | 2 +- test-data/unit/check-typeddict.test | 17 +++++++++++++++ 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/mypy/argmap.py b/mypy/argmap.py index 2f786bf17d28..40c715071971 100644 --- a/mypy/argmap.py +++ b/mypy/argmap.py @@ -2,7 +2,7 @@ from typing import List, Optional, Sequence, Callable -from mypy.types import Type, Instance, TupleType, AnyType, TypeOfAny +from mypy.types import Type, Instance, TupleType, AnyType, TypeOfAny, TypedDictType from mypy import nodes @@ -65,13 +65,23 @@ def map_actuals_to_formals(caller_kinds: List[int], map[callee_kinds.index(nodes.ARG_STAR2)].append(i) else: assert kind == nodes.ARG_STAR2 - for j in range(ncallee): - # TODO tuple varargs complicate this - no_certain_match = ( - not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) - if ((callee_names[j] and no_certain_match) - or callee_kinds[j] == nodes.ARG_STAR2): - map[j].append(i) + argt = caller_arg_type(i) + if isinstance(argt, TypedDictType): + for name, value in argt.items.items(): + if name in callee_names: + map[callee_names.index(name)].append(i) + elif nodes.ARG_STAR2 in callee_kinds: + map[callee_kinds.index(nodes.ARG_STAR2)].append(i) + else: + # We don't exactly which **kwargs are provided by the + # caller. Assume that they will fill the remaining arguments. + for j in range(ncallee): + # TODO tuple varargs complicate this + no_certain_match = ( + not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) + if ((callee_names[j] and no_certain_match) + or callee_kinds[j] == nodes.ARG_STAR2): + map[j].append(i) return map @@ -95,13 +105,12 @@ def map_formals_to_actuals(caller_kinds: List[int], return actual_to_formal -def get_actual_type(arg_type: Type, kind: int, +def get_actual_type(arg_type: Type, kind: int, arg_name: Optional[str], tuple_counter: List[int]) -> Type: """Return the type of an actual argument with the given kind. If the argument is a *arg, return the individual argument item. """ - if kind == nodes.ARG_STAR: if isinstance(arg_type, Instance): if arg_type.type.fullname() == 'builtins.list': @@ -119,7 +128,10 @@ def get_actual_type(arg_type: Type, kind: int, else: return AnyType(TypeOfAny.from_error) elif kind == nodes.ARG_STAR2: - if isinstance(arg_type, Instance) and (arg_type.type.fullname() == 'builtins.dict'): + if isinstance(arg_type, TypedDictType): + assert arg_name is not None + return arg_type.items[arg_name] + elif isinstance(arg_type, Instance) and (arg_type.type.fullname() == 'builtins.dict'): # Dict **arg. TODO more general (Mapping) return arg_type.args[1] else: diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index d049098ff1bb..26e4545dfb93 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -704,8 +704,8 @@ def check_call(self, callee: Type, args: List[Expression], self.check_argument_count(callee, arg_types, arg_kinds, arg_names, formal_to_actual, context, self.msg) - self.check_argument_types(arg_types, arg_kinds, callee, - formal_to_actual, context, + self.check_argument_types(arg_types, arg_kinds, callee.arg_names, + callee, formal_to_actual, context, messages=arg_messages) if (callee.is_type_obj() and (len(arg_types) == 1) @@ -1139,7 +1139,10 @@ def check_argument_count(self, callee: CallableType, actual_types: List[Type], ok = False return ok - def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], + def check_argument_types(self, + arg_types: List[Type], + arg_kinds: List[int], + formal_names: List[Optional[str]], callee: CallableType, formal_to_actual: List[List[int]], context: Context, @@ -1173,7 +1176,7 @@ def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], and arg_kinds[actual] == nodes.ARG_STAR): # The tuple is exhausted. Continue with further arguments. continue - actual_type = get_actual_type(arg_type, arg_kinds[actual], + actual_type = get_actual_type(arg_type, arg_kinds[actual], formal_names[i], tuple_counter) check_arg(actual_type, arg_type, arg_kinds[actual], callee.arg_types[i], @@ -1188,6 +1191,7 @@ def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], while tuple_counter[0] < len(tuplet.items): actual_type = get_actual_type(arg_type, arg_kinds[actual], + callee.arg_names[i], tuple_counter) check_arg(actual_type, arg_type, arg_kinds[actual], callee.arg_types[i], @@ -1669,8 +1673,8 @@ def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, raise Finished try: - self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, - context=context, check_arg=check_arg) + self.check_argument_types(arg_types, arg_kinds, callee.arg_names, callee, + formal_to_actual, context=context, check_arg=check_arg) return True except Finished: return False diff --git a/mypy/constraints.py b/mypy/constraints.py index 5f6ddf1769b3..a04ba9e0a360 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -63,7 +63,7 @@ def infer_constraints_for_callable( continue actual_type = get_actual_type(actual_arg_type, arg_kinds[actual], - tuple_counter) + callee.arg_names, tuple_counter) c = infer_constraints(callee.arg_types[i], actual_type, SUPERTYPE_OF) constraints.extend(c) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index f1734907e8e3..7598a7a9abe5 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1413,3 +1413,20 @@ from mypy_extensions import TypedDict tp = TypedDict('tp', {'x': int}) [builtins fixtures/dict.pyi] [out] + +[case testTypedDictAsStarStarArg] +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': str}) + +def f(x: int, y: str) -> None: ... +def g(x: int, y: int) -> None: ... + +a: A +f(**a) +g(**a) # E: Argument 1 to "g" has incompatible type "**A"; expected "int" + +# TODO: +# - missing arguments +# - too many arguments +# - caller takes **kwargs From d921d818290040f731d4db95127dedac1aa5d3f1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 16 Nov 2018 17:03:33 +0000 Subject: [PATCH 02/17] Improve error message --- mypy/messages.py | 10 ++++++++++ test-data/unit/check-typeddict.test | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/mypy/messages.py b/mypy/messages.py index 2162c06acbf0..bf1e4bc02db1 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -671,6 +671,16 @@ def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: if arg_name is not None: arg_label = '"{}"'.format(arg_name) + if (arg_kind == ARG_STAR2 + and isinstance(arg_type, TypedDictType) + and m <= len(callee.arg_names) + and callee.arg_names[m - 1] is not None): + arg_name = callee.arg_names[m - 1] + arg_type_str, expected_type_str = self.format_distinctly( + arg_type.items[arg_name], + expected_type, + bare=True) + arg_label = '"{}"'.format(arg_name) msg = 'Argument {} {}has incompatible type {}; expected {}'.format( arg_label, target, self.quote_type_string(arg_type_str), self.quote_type_string(expected_type_str)) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 7598a7a9abe5..3998c48d3823 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1418,13 +1418,16 @@ tp = TypedDict('tp', {'x': int}) from mypy_extensions import TypedDict A = TypedDict('A', {'x': int, 'y': str}) +class B: pass def f(x: int, y: str) -> None: ... def g(x: int, y: int) -> None: ... +def h(x: B, y: str) -> None: ... a: A f(**a) -g(**a) # E: Argument 1 to "g" has incompatible type "**A"; expected "int" +g(**a) # E: Argument "y" to "g" has incompatible type "str"; expected "int" +h(**a) # E: Argument "x" to "h" has incompatible type "int"; expected "B" # TODO: # - missing arguments From e71b5c48dd95a8eb838afe5273e7cd651f8a6d26 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 16 Nov 2018 18:34:11 +0000 Subject: [PATCH 03/17] Check for too many arguments --- mypy/checkexpr.py | 14 +++++++++++++- test-data/unit/check-typeddict.test | 19 +++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 26e4545dfb93..2316bad26cc7 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1049,7 +1049,9 @@ def apply_inferred_arguments(self, callee_type: CallableType, # arguments. return self.apply_generic_arguments(callee_type, inferred_args, context) - def check_argument_count(self, callee: CallableType, actual_types: List[Type], + def check_argument_count(self, + callee: CallableType, + actual_types: List[Type], actual_kinds: List[int], actual_names: Optional[Sequence[Optional[str]]], formal_to_actual: List[List[int]], @@ -1072,6 +1074,8 @@ def check_argument_count(self, callee: CallableType, actual_types: List[Type], is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors. ok = True # False if we've found any error. + + # Check for extra actuals. for i, kind in enumerate(actual_kinds): if i not in all_actuals and ( kind != nodes.ARG_STAR or @@ -1103,7 +1107,15 @@ def check_argument_count(self, callee: CallableType, actual_types: List[Type], ok = False # *args can be applied even if the function takes a fixed # number of positional arguments. This may succeed at runtime. + elif kind == nodes.ARG_STAR2 and isinstance(actual_types[i], TypedDictType): + if all_actuals.count(i) < len(actual_types[i].items): + # Too many tuple items as some did not match. + if messages: + assert context, "Internal error: messages given without context" + messages.too_many_arguments(callee, context) + ok = False + # Check for too many or few values for formals. for i, kind in enumerate(formal_kinds): if kind == nodes.ARG_POS and (not formal_to_actual[i] and not is_unexpected_arg_error): diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 3998c48d3823..b90e6d91cfe8 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1420,16 +1420,19 @@ from mypy_extensions import TypedDict A = TypedDict('A', {'x': int, 'y': str}) class B: pass -def f(x: int, y: str) -> None: ... -def g(x: int, y: int) -> None: ... -def h(x: B, y: str) -> None: ... +def f1(x: int, y: str) -> None: ... +def f2(x: int, y: int) -> None: ... +def f3(x: B, y: str) -> None: ... +def f4(x: int) -> None: pass +def f5(x: int, y: str, z: int) -> None: pass a: A -f(**a) -g(**a) # E: Argument "y" to "g" has incompatible type "str"; expected "int" -h(**a) # E: Argument "x" to "h" has incompatible type "int"; expected "B" +f1(**a) +f2(**a) # E: Argument "y" to "f2" has incompatible type "str"; expected "int" +f3(**a) # E: Argument "x" to "f3" has incompatible type "int"; expected "B" +f4(**a) # E: Too many arguments for "f4" +f5(**a) # E: Too few arguments for "f5" # TODO: -# - missing arguments -# - too many arguments +# - duplicate args due to **a # - caller takes **kwargs From 3284026333dda2e7d2e20c8b9a159ed18654ace3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 16 Nov 2018 18:44:54 +0000 Subject: [PATCH 04/17] Fix error --- mypy/constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index a04ba9e0a360..7c0ef3e7c031 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -63,7 +63,7 @@ def infer_constraints_for_callable( continue actual_type = get_actual_type(actual_arg_type, arg_kinds[actual], - callee.arg_names, tuple_counter) + callee.arg_names[i], tuple_counter) c = infer_constraints(callee.arg_types[i], actual_type, SUPERTYPE_OF) constraints.extend(c) From ca602c206d795bedcf0a9dfc152fea12962d8da0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 16 Nov 2018 18:45:03 +0000 Subject: [PATCH 05/17] Fix type check --- mypy/messages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/messages.py b/mypy/messages.py index bf1e4bc02db1..f0d00c876bac 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -676,6 +676,7 @@ def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: and m <= len(callee.arg_names) and callee.arg_names[m - 1] is not None): arg_name = callee.arg_names[m - 1] + assert arg_name is not None arg_type_str, expected_type_str = self.format_distinctly( arg_type.items[arg_name], expected_type, From b03e6f906115076e556c607f9e7dda3c86651ffc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 16 Nov 2018 18:45:21 +0000 Subject: [PATCH 06/17] Minor refactoring --- mypy/checkexpr.py | 30 ++++++++++------------------- test-data/unit/check-typeddict.test | 1 + 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 2316bad26cc7..7c91537e55a0 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1064,6 +1064,11 @@ def check_argument_count(self, Return False if there were any errors. Otherwise return True """ + if messages: + assert context, "Internal error: messages gives without context" + elif context is None: + context = TempNode(AnyType(TypeOfAny.special_form)) # Avoid "is None" checks + # TODO(jukka): We could return as soon as we find an error if messages is None. formal_kinds = callee.arg_kinds @@ -1084,36 +1089,25 @@ def check_argument_count(self, ok = False if kind != nodes.ARG_NAMED: if messages: - assert context, "Internal error: messages given without context" messages.too_many_arguments(callee, context) else: if messages: - assert context, "Internal error: messages given without context" assert actual_names, "Internal error: named kinds without names given" act_name = actual_names[i] assert act_name is not None - messages.unexpected_keyword_argument( - callee, act_name, context) + messages.unexpected_keyword_argument(callee, act_name, context) is_unexpected_arg_error = True - elif kind == nodes.ARG_STAR and ( - nodes.ARG_STAR not in formal_kinds): + elif ((kind == nodes.ARG_STAR and nodes.ARG_STAR not in formal_kinds) + or kind == nodes.ARG_STAR2): actual_type = actual_types[i] - if isinstance(actual_type, TupleType): + if isinstance(actual_type, (TupleType, TypedDictType)): if all_actuals.count(i) < len(actual_type.items): # Too many tuple items as some did not match. if messages: - assert context, "Internal error: messages given without context" messages.too_many_arguments(callee, context) ok = False - # *args can be applied even if the function takes a fixed + # *args/**kwargs can be applied even if the function takes a fixed # number of positional arguments. This may succeed at runtime. - elif kind == nodes.ARG_STAR2 and isinstance(actual_types[i], TypedDictType): - if all_actuals.count(i) < len(actual_types[i].items): - # Too many tuple items as some did not match. - if messages: - assert context, "Internal error: messages given without context" - messages.too_many_arguments(callee, context) - ok = False # Check for too many or few values for formals. for i, kind in enumerate(formal_kinds): @@ -1121,7 +1115,6 @@ def check_argument_count(self, not is_unexpected_arg_error): # No actual for a mandatory positional formal. if messages: - assert context, "Internal error: messages given without context" messages.too_few_arguments(callee, context, actual_names) ok = False elif kind == nodes.ARG_NAMED and (not formal_to_actual[i] and @@ -1130,7 +1123,6 @@ def check_argument_count(self, if messages: argname = callee.arg_names[i] assert argname is not None - assert context, "Internal error: messages given without context" messages.missing_named_argument(callee, context, argname) ok = False elif kind in [nodes.ARG_POS, nodes.ARG_OPT, @@ -1139,14 +1131,12 @@ def check_argument_count(self, if (self.chk.in_checked_function() or isinstance(actual_types[formal_to_actual[i][0]], TupleType)): if messages: - assert context, "Internal error: messages given without context" messages.duplicate_argument_value(callee, i, context) ok = False elif (kind in (nodes.ARG_NAMED, nodes.ARG_NAMED_OPT) and formal_to_actual[i] and actual_kinds[formal_to_actual[i][0]] not in [nodes.ARG_NAMED, nodes.ARG_STAR2]): # Positional argument when expecting a keyword argument. if messages: - assert context, "Internal error: messages given without context" messages.too_many_positional_arguments(callee, context) ok = False return ok diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index b90e6d91cfe8..980769246d2e 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1434,5 +1434,6 @@ f4(**a) # E: Too many arguments for "f4" f5(**a) # E: Too few arguments for "f5" # TODO: +# - constraint inference # - duplicate args due to **a # - caller takes **kwargs From 8e914ed079b29deb558bb394cdd09477006a1b5e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Nov 2018 13:52:22 +0000 Subject: [PATCH 07/17] Improve error messages and refactor --- mypy/checkexpr.py | 89 ++++++++++++++++++----------- mypy/messages.py | 14 +++++ test-data/unit/check-typeddict.test | 8 ++- 3 files changed, 76 insertions(+), 35 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 7c91537e55a0..73b3968f4b75 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1070,47 +1070,17 @@ def check_argument_count(self, context = TempNode(AnyType(TypeOfAny.special_form)) # Avoid "is None" checks # TODO(jukka): We could return as soon as we find an error if messages is None. - formal_kinds = callee.arg_kinds # Collect list of all actual arguments matched to formal arguments. all_actuals = [] # type: List[int] for actuals in formal_to_actual: all_actuals.extend(actuals) - is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors. - ok = True # False if we've found any error. - - # Check for extra actuals. - for i, kind in enumerate(actual_kinds): - if i not in all_actuals and ( - kind != nodes.ARG_STAR or - not is_empty_tuple(actual_types[i])): - # Extra actual: not matched by a formal argument. - ok = False - if kind != nodes.ARG_NAMED: - if messages: - messages.too_many_arguments(callee, context) - else: - if messages: - assert actual_names, "Internal error: named kinds without names given" - act_name = actual_names[i] - assert act_name is not None - messages.unexpected_keyword_argument(callee, act_name, context) - is_unexpected_arg_error = True - elif ((kind == nodes.ARG_STAR and nodes.ARG_STAR not in formal_kinds) - or kind == nodes.ARG_STAR2): - actual_type = actual_types[i] - if isinstance(actual_type, (TupleType, TypedDictType)): - if all_actuals.count(i) < len(actual_type.items): - # Too many tuple items as some did not match. - if messages: - messages.too_many_arguments(callee, context) - ok = False - # *args/**kwargs can be applied even if the function takes a fixed - # number of positional arguments. This may succeed at runtime. + ok, is_unexpected_arg_error = self.check_for_extra_actual_arguments( + callee, actual_types, actual_kinds, actual_names, all_actuals, context, messages) # Check for too many or few values for formals. - for i, kind in enumerate(formal_kinds): + for i, kind in enumerate(callee.arg_kinds): if kind == nodes.ARG_POS and (not formal_to_actual[i] and not is_unexpected_arg_error): # No actual for a mandatory positional formal. @@ -1141,6 +1111,59 @@ def check_argument_count(self, ok = False return ok + def check_for_extra_actual_arguments(self, + callee: CallableType, + actual_types: List[Type], + actual_kinds: List[int], + actual_names: Optional[Sequence[Optional[str]]], + all_actuals: List[int], + context: Context, + messages: Optional[MessageBuilder]) -> Tuple[bool, bool]: + """Check for extra actual arguments. + + Return tuple (was everything ok, + was there an extra keyword argument error). + """ + + is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors + ok = True # False if we've found any error + + for i, kind in enumerate(actual_kinds): + if i not in all_actuals and ( + kind != nodes.ARG_STAR or + not is_empty_tuple(actual_types[i])): + # Extra actual: not matched by a formal argument. + ok = False + if kind != nodes.ARG_NAMED: + if messages: + messages.too_many_arguments(callee, context) + else: + if messages: + assert actual_names, "Internal error: named kinds without names given" + act_name = actual_names[i] + assert act_name is not None + messages.unexpected_keyword_argument(callee, act_name, context) + is_unexpected_arg_error = True + elif ((kind == nodes.ARG_STAR and nodes.ARG_STAR not in callee.arg_kinds) + or kind == nodes.ARG_STAR2): + actual_type = actual_types[i] + if isinstance(actual_type, (TupleType, TypedDictType)): + if all_actuals.count(i) < len(actual_type.items): + # Too many tuple/dict items as some did not match. + if messages: + if (kind != nodes.ARG_STAR2 + or not isinstance(actual_type, TypedDictType)): + messages.too_many_arguments(callee, context) + else: + messages.too_many_arguments_from_typed_dict(callee, actual_type, + context) + is_unexpected_arg_error = True + ok = False + # *args/**kwargs can be applied even if the function takes a fixed + # number of positional arguments. This may succeed at runtime. + + return ok, is_unexpected_arg_error + def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], diff --git a/mypy/messages.py b/mypy/messages.py index f0d00c876bac..9d0a5ee2d55e 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -722,6 +722,20 @@ def too_many_arguments(self, callee: CallableType, context: Context) -> None: msg = 'Too many arguments' + for_function(callee) self.fail(msg, context) + def too_many_arguments_from_typed_dict(self, + callee: CallableType, + arg_type: TypedDictType, + context: Context) -> None: + # Try to determine the name of the extra argument. + for key in arg_type.items: + if key not in callee.arg_names: + msg = 'Extra argument "{}" from **args'.format(key) + for_function(callee) + break + else: + self.too_many_arguments(callee, context) + return + self.fail(msg, context) + def too_many_positional_arguments(self, callee: CallableType, context: Context) -> None: msg = 'Too many positional arguments' + for_function(callee) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 980769246d2e..21a0bf42243e 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1425,15 +1425,19 @@ def f2(x: int, y: int) -> None: ... def f3(x: B, y: str) -> None: ... def f4(x: int) -> None: pass def f5(x: int, y: str, z: int) -> None: pass +def f6(x: int, z: str) -> None: pass a: A f1(**a) f2(**a) # E: Argument "y" to "f2" has incompatible type "str"; expected "int" f3(**a) # E: Argument "x" to "f3" has incompatible type "int"; expected "B" -f4(**a) # E: Too many arguments for "f4" +f4(**a) # E: Extra argument "y" from **args for "f4" f5(**a) # E: Too few arguments for "f5" +f6(**a) # E: Extra argument "y" from **args for "f6" +f1(1, **a) # E: "f1" gets multiple values for keyword argument "x" + # TODO: # - constraint inference -# - duplicate args due to **a # - caller takes **kwargs +# ? non-total typeddict shouldn't generate extra arg errors From 95bba09aa1c1c5e53732236af1aed2fabc284494 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Nov 2018 13:57:07 +0000 Subject: [PATCH 08/17] Add test case --- test-data/unit/check-typeddict.test | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 21a0bf42243e..592b554f16c4 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1436,8 +1436,14 @@ f5(**a) # E: Too few arguments for "f5" f6(**a) # E: Extra argument "y" from **args for "f6" f1(1, **a) # E: "f1" gets multiple values for keyword argument "x" +[case testTypedDictAsStarStarArgConstraints] +from typing import TypeVar, Union +from mypy_extensions import TypedDict + +T = TypeVar('T') +S = TypeVar('S') +def f1(x: T, y: S) -> Union[T, S]: ... -# TODO: -# - constraint inference -# - caller takes **kwargs -# ? non-total typeddict shouldn't generate extra arg errors +A = TypedDict('A', {'y': int, 'x': str}) +a: A +reveal_type(f1(**a)) # E: Revealed type is 'Union[builtins.str*, builtins.int*]' From ed68a7461c75df5cd84109a6f354ee739f6f58c3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Nov 2018 15:02:22 +0000 Subject: [PATCH 09/17] Fix handling of callee **kwargs --- mypy/argmap.py | 85 ++++++++++++++++++----------- mypy/checkexpr.py | 28 +++++----- mypy/constraints.py | 15 ++--- mypy/messages.py | 3 +- test-data/unit/check-typeddict.test | 20 +++++++ 5 files changed, 98 insertions(+), 53 deletions(-) diff --git a/mypy/argmap.py b/mypy/argmap.py index 40c715071971..9fccf9b9254a 100644 --- a/mypy/argmap.py +++ b/mypy/argmap.py @@ -1,6 +1,6 @@ """Utilities for mapping between actual and formal arguments (and their types).""" -from typing import List, Optional, Sequence, Callable +from typing import List, Optional, Sequence, Callable, Set from mypy.types import Type, Instance, TupleType, AnyType, TypeOfAny, TypedDictType from mypy import nodes @@ -105,37 +105,60 @@ def map_formals_to_actuals(caller_kinds: List[int], return actual_to_formal -def get_actual_type(arg_type: Type, kind: int, arg_name: Optional[str], - tuple_counter: List[int]) -> Type: - """Return the type of an actual argument with the given kind. +class ArgTypeMapper: + """Utility class for mapping actual argument types to formal argument types. - If the argument is a *arg, return the individual argument item. + The main job is to expand tuple *args and typed dict **kwargs in caller, and to + keep track of which tuple/typed dict items have already been consumed. """ - if kind == nodes.ARG_STAR: - if isinstance(arg_type, Instance): - if arg_type.type.fullname() == 'builtins.list': - # List *arg. - return arg_type.args[0] - elif arg_type.args: - # TODO try to map type arguments to Iterable - return arg_type.args[0] + + def __init__(self) -> None: + # Next tuple *args index to use. + self.tuple_index = 0 + # Keyword arguments in TypedDict **kwargs used. + self.kwargs_used = set() # type: Set[str] + + def get_actual_type(self, arg_type: Type, kind: int, arg_name: Optional[str]) -> List[Type]: + """Return the type(s) of an actual argument with the given kind. + + If the argument is a *args, return the individual argument item. The + tuple_counter argument tracks the next unused tuple item. + + If the argument is a **kwargs, return the item type based on argument name, + or all item types otherwise. + """ + if kind == nodes.ARG_STAR: + if isinstance(arg_type, Instance): + if arg_type.type.fullname() == 'builtins.list': + # List *arg. + return [arg_type.args[0]] + elif arg_type.args: + # TODO try to map type arguments to Iterable + return [arg_type.args[0]] + else: + return [AnyType(TypeOfAny.from_error)] + elif isinstance(arg_type, TupleType): + # Get the next tuple item of a tuple *arg. + self.tuple_index += 1 + return [arg_type.items[self.tuple_index - 1]] else: - return AnyType(TypeOfAny.from_error) - elif isinstance(arg_type, TupleType): - # Get the next tuple item of a tuple *arg. - tuple_counter[0] += 1 - return arg_type.items[tuple_counter[0] - 1] - else: - return AnyType(TypeOfAny.from_error) - elif kind == nodes.ARG_STAR2: - if isinstance(arg_type, TypedDictType): - assert arg_name is not None - return arg_type.items[arg_name] - elif isinstance(arg_type, Instance) and (arg_type.type.fullname() == 'builtins.dict'): - # Dict **arg. TODO more general (Mapping) - return arg_type.args[1] + return [AnyType(TypeOfAny.from_error)] + elif kind == nodes.ARG_STAR2: + if isinstance(arg_type, TypedDictType): + if arg_name in arg_type.items: + # Lookup type based on keyword argument name. + assert arg_name is not None + self.kwargs_used.add(arg_name) + return [arg_type.items[arg_name]] + else: + # Callee takes **kwargs. Give all remaining keyword args. + return [value for key, value in arg_type.items.items() + if key not in self.kwargs_used] + elif isinstance(arg_type, Instance) and (arg_type.type.fullname() == 'builtins.dict'): + # Dict **arg. TODO more general (Mapping) + return [arg_type.args[1]] + else: + return [AnyType(TypeOfAny.from_error)] else: - return AnyType(TypeOfAny.from_error) - else: - # No translation for other kinds. - return arg_type + # No translation for other kinds. + return [arg_type] diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 73b3968f4b75..982f776818e0 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -51,7 +51,7 @@ from mypy import applytype from mypy import erasetype from mypy.checkmember import analyze_member_access, type_object_type -from mypy.argmap import get_actual_type, map_actuals_to_formals, map_formals_to_actuals +from mypy.argmap import ArgTypeMapper, map_actuals_to_formals, map_formals_to_actuals from mypy.checkstrformat import StringFormatterChecker from mypy.expandtype import expand_type, expand_type_by_instance, freshen_function_type_vars from mypy.util import split_module_names @@ -1180,7 +1180,7 @@ def check_argument_types(self, messages = messages or self.msg check_arg = check_arg or self.check_arg # Keep track of consumed tuple *arg items. - tuple_counter = [0] + mapper = ArgTypeMapper() for i, actuals in enumerate(formal_to_actual): for actual in actuals: arg_type = arg_types[actual] @@ -1197,15 +1197,15 @@ def check_argument_types(self, # Get the type of an individual actual argument (for *args # and **args this is the item type, not the collection type). if (isinstance(arg_type, TupleType) - and tuple_counter[0] >= len(arg_type.items) + and mapper.tuple_index >= len(arg_type.items) and arg_kinds[actual] == nodes.ARG_STAR): # The tuple is exhausted. Continue with further arguments. continue - actual_type = get_actual_type(arg_type, arg_kinds[actual], formal_names[i], - tuple_counter) - check_arg(actual_type, arg_type, arg_kinds[actual], - callee.arg_types[i], - actual + 1, i + 1, callee, context, messages) + actual_types = mapper.get_actual_type(arg_type, arg_kinds[actual], formal_names[i]) + for actual_type in actual_types: + check_arg(actual_type, arg_type, arg_kinds[actual], + callee.arg_types[i], + actual + 1, i + 1, callee, context, messages) # There may be some remaining tuple varargs items that haven't # been checked yet. Handle them. @@ -1213,12 +1213,12 @@ def check_argument_types(self, if (callee.arg_kinds[i] == nodes.ARG_STAR and arg_kinds[actual] == nodes.ARG_STAR and isinstance(tuplet, TupleType)): - while tuple_counter[0] < len(tuplet.items): - actual_type = get_actual_type(arg_type, - arg_kinds[actual], - callee.arg_names[i], - tuple_counter) - check_arg(actual_type, arg_type, arg_kinds[actual], + while mapper.tuple_index < len(tuplet.items): + actual_types = mapper.get_actual_type(arg_type, + arg_kinds[actual], + callee.arg_names[i]) + assert len(actual_types) == 1 + check_arg(actual_types[0], arg_type, arg_kinds[actual], callee.arg_types[i], actual + 1, i + 1, callee, context, messages) diff --git a/mypy/constraints.py b/mypy/constraints.py index 7c0ef3e7c031..09fdade801ed 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -13,7 +13,7 @@ from mypy.sametypes import is_same_type from mypy.erasetype import erase_typevars from mypy.nodes import COVARIANT, CONTRAVARIANT -from mypy.argmap import get_actual_type +from mypy.argmap import ArgTypeMapper MYPY = False if MYPY: @@ -54,7 +54,7 @@ def infer_constraints_for_callable( Return a list of constraints. """ constraints = [] # type: List[Constraint] - tuple_counter = [0] + mapper = ArgTypeMapper() for i, actuals in enumerate(formal_to_actual): for actual in actuals: @@ -62,11 +62,12 @@ def infer_constraints_for_callable( if actual_arg_type is None: continue - actual_type = get_actual_type(actual_arg_type, arg_kinds[actual], - callee.arg_names[i], tuple_counter) - c = infer_constraints(callee.arg_types[i], actual_type, - SUPERTYPE_OF) - constraints.extend(c) + actual_types = mapper.get_actual_type(actual_arg_type, arg_kinds[actual], + callee.arg_names[i]) + for actual_type in actual_types: + c = infer_constraints(callee.arg_types[i], actual_type, + SUPERTYPE_OF) + constraints.extend(c) return constraints diff --git a/mypy/messages.py b/mypy/messages.py index 9d0a5ee2d55e..d2d05e60d228 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -674,7 +674,8 @@ def incompatible_argument(self, n: int, m: int, callee: CallableType, arg_type: if (arg_kind == ARG_STAR2 and isinstance(arg_type, TypedDictType) and m <= len(callee.arg_names) - and callee.arg_names[m - 1] is not None): + and callee.arg_names[m - 1] is not None + and callee.arg_kinds[m - 1] != ARG_STAR2): arg_name = callee.arg_names[m - 1] assert arg_name is not None arg_type_str, expected_type_str = self.format_distinctly( diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 592b554f16c4..74f24638bb7b 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1447,3 +1447,23 @@ def f1(x: T, y: S) -> Union[T, S]: ... A = TypedDict('A', {'y': int, 'x': str}) a: A reveal_type(f1(**a)) # E: Revealed type is 'Union[builtins.str*, builtins.int*]' + +[case testTypedDictAsStarStarArgCalleeKwargs] +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': str}) +B = TypedDict('B', {'x': str, 'y': str}) + +def f(**kwargs: str) -> None: ... +def g(x: int, **kwargs: str) -> None: ... + +a: A +b: B +f(**a) # E: Argument 1 to "f" has incompatible type "**A"; expected "str" +f(**b) +g(**a) +g(**b) # E: Argument "x" to "g" has incompatible type "str"; expected "int" +g(1, **a) # E: "g" gets multiple values for keyword argument "x" +g(1, **b) # E: "g" gets multiple values for keyword argument "x" \ + # E: Argument "x" to "g" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] From 6082dc9e0879a230de5fef51d99945c48feeee0c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Nov 2018 12:13:37 +0000 Subject: [PATCH 10/17] Refactor formal/actual argument mapping --- mypy/argmap.py | 93 +++++++++++++++++++++++++++++---------------- mypy/checkexpr.py | 59 +++++++++------------------- mypy/constraints.py | 8 ++-- 3 files changed, 83 insertions(+), 77 deletions(-) diff --git a/mypy/argmap.py b/mypy/argmap.py index 9fccf9b9254a..cf1323f005ce 100644 --- a/mypy/argmap.py +++ b/mypy/argmap.py @@ -105,11 +105,25 @@ def map_formals_to_actuals(caller_kinds: List[int], return actual_to_formal -class ArgTypeMapper: - """Utility class for mapping actual argument types to formal argument types. +class ArgTypeExpander: + """Utility class for mapping actual argument types to formal arguments. - The main job is to expand tuple *args and typed dict **kwargs in caller, and to - keep track of which tuple/typed dict items have already been consumed. + One of the main responsibilities is to expand caller tuple *args and TypedDict + **kwargs, and to keep track of which tuple/TypedDict items have already been + consumed. + + Example: + + . def f(x: int, *args: str) -> None: ... + . f(*(1, 'x', 1.1) + + We'd call expand_actual_type twice: + + 1. The first call would provide 'int' as the type of 'x'. + 2. The second call would provide 'str' and 'float' as the types of '*args'. + + Construct a separate instance for each call since instances have per-call + state. """ def __init__(self) -> None: @@ -118,47 +132,60 @@ def __init__(self) -> None: # Keyword arguments in TypedDict **kwargs used. self.kwargs_used = set() # type: Set[str] - def get_actual_type(self, arg_type: Type, kind: int, arg_name: Optional[str]) -> List[Type]: - """Return the type(s) of an actual argument with the given kind. + def expand_actual_type(self, + actual_type: Type, + actual_kind: int, + formal_name: Optional[str], + formal_kind: int) -> List[Type]: + """Return the actual (caller) type(s) of a formal argument with the given kinds. + + If the actual argument is a tuple *args, return the individual tuple item(s) that + map(s) to the formal arg. - If the argument is a *args, return the individual argument item. The - tuple_counter argument tracks the next unused tuple item. + If the actual argument is a TypedDict **kwargs, return the matching typed dict dict + value type(s) based on formal argument name and kind. - If the argument is a **kwargs, return the item type based on argument name, - or all item types otherwise. + This is supposed to be called for each formal, in order. Call multiple times per + formal if multiple actuals map to a formal. """ - if kind == nodes.ARG_STAR: - if isinstance(arg_type, Instance): - if arg_type.type.fullname() == 'builtins.list': + if actual_kind == nodes.ARG_STAR: + if isinstance(actual_type, Instance): + if actual_type.type.fullname() == 'builtins.list': # List *arg. - return [arg_type.args[0]] - elif arg_type.args: - # TODO try to map type arguments to Iterable - return [arg_type.args[0]] + return [actual_type.args[0]] + elif actual_type.args: + # TODO: Try to map type arguments to Iterable + return [actual_type.args[0]] else: return [AnyType(TypeOfAny.from_error)] - elif isinstance(arg_type, TupleType): + elif isinstance(actual_type, TupleType): # Get the next tuple item of a tuple *arg. - self.tuple_index += 1 - return [arg_type.items[self.tuple_index - 1]] + if formal_kind != nodes.ARG_STAR: + self.tuple_index += 1 + return [actual_type.items[self.tuple_index - 1]] + else: + # Callee takes *args. Give all remaining actual *args. + return actual_type.items[self.tuple_index:] else: return [AnyType(TypeOfAny.from_error)] - elif kind == nodes.ARG_STAR2: - if isinstance(arg_type, TypedDictType): - if arg_name in arg_type.items: + elif actual_kind == nodes.ARG_STAR2: + if isinstance(actual_type, TypedDictType): + if formal_kind != nodes.ARG_STAR2 and formal_name in actual_type.items: # Lookup type based on keyword argument name. - assert arg_name is not None - self.kwargs_used.add(arg_name) - return [arg_type.items[arg_name]] + assert formal_name is not None + self.kwargs_used.add(formal_name) + return [actual_type.items[formal_name]] else: - # Callee takes **kwargs. Give all remaining keyword args. - return [value for key, value in arg_type.items.items() + # Callee takes **kwargs. Give all remaining actual **kwargs. + return [value for key, value in actual_type.items.items() if key not in self.kwargs_used] - elif isinstance(arg_type, Instance) and (arg_type.type.fullname() == 'builtins.dict'): - # Dict **arg. TODO more general (Mapping) - return [arg_type.args[1]] + elif (isinstance(actual_type, Instance) + and (actual_type.type.fullname() == 'builtins.dict')): + # Dict **arg. + # TODO: Handle arbitrary Mapping + return [actual_type.args[1]] else: return [AnyType(TypeOfAny.from_error)] else: - # No translation for other kinds. - return [arg_type] + # No translation for other kinds -- 1:1 mapping. + return [actual_type] diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 982f776818e0..0ad883d34e28 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -51,7 +51,7 @@ from mypy import applytype from mypy import erasetype from mypy.checkmember import analyze_member_access, type_object_type -from mypy.argmap import ArgTypeMapper, map_actuals_to_formals, map_formals_to_actuals +from mypy.argmap import ArgTypeExpander, map_actuals_to_formals, map_formals_to_actuals from mypy.checkstrformat import StringFormatterChecker from mypy.expandtype import expand_type, expand_type_by_instance, freshen_function_type_vars from mypy.util import split_module_names @@ -704,8 +704,7 @@ def check_call(self, callee: Type, args: List[Expression], self.check_argument_count(callee, arg_types, arg_kinds, arg_names, formal_to_actual, context, self.msg) - self.check_argument_types(arg_types, arg_kinds, callee.arg_names, - callee, formal_to_actual, context, + self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, context, messages=arg_messages) if (callee.is_type_obj() and (len(arg_types) == 1) @@ -1167,7 +1166,6 @@ def check_for_extra_actual_arguments(self, def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], - formal_names: List[Optional[str]], callee: CallableType, formal_to_actual: List[List[int]], context: Context, @@ -1180,48 +1178,29 @@ def check_argument_types(self, messages = messages or self.msg check_arg = check_arg or self.check_arg # Keep track of consumed tuple *arg items. - mapper = ArgTypeMapper() + mapper = ArgTypeExpander() for i, actuals in enumerate(formal_to_actual): for actual in actuals: - arg_type = arg_types[actual] - if arg_type is None: + actual_type = arg_types[actual] + if actual_type is None: continue # Some kind of error was already reported. + actual_kind = arg_kinds[actual] # Check that a *arg is valid as varargs. - if (arg_kinds[actual] == nodes.ARG_STAR and - not self.is_valid_var_arg(arg_type)): - messages.invalid_var_arg(arg_type, context) - if (arg_kinds[actual] == nodes.ARG_STAR2 and - not self.is_valid_keyword_var_arg(arg_type)): - is_mapping = is_subtype(arg_type, self.chk.named_type('typing.Mapping')) - messages.invalid_keyword_var_arg(arg_type, is_mapping, context) - # Get the type of an individual actual argument (for *args - # and **args this is the item type, not the collection type). - if (isinstance(arg_type, TupleType) - and mapper.tuple_index >= len(arg_type.items) - and arg_kinds[actual] == nodes.ARG_STAR): - # The tuple is exhausted. Continue with further arguments. - continue - actual_types = mapper.get_actual_type(arg_type, arg_kinds[actual], formal_names[i]) - for actual_type in actual_types: - check_arg(actual_type, arg_type, arg_kinds[actual], + if (actual_kind == nodes.ARG_STAR and + not self.is_valid_var_arg(actual_type)): + messages.invalid_var_arg(actual_type, context) + if (actual_kind == nodes.ARG_STAR2 and + not self.is_valid_keyword_var_arg(actual_type)): + is_mapping = is_subtype(actual_type, self.chk.named_type('typing.Mapping')) + messages.invalid_keyword_var_arg(actual_type, is_mapping, context) + expanded_actuals = mapper.expand_actual_type( + actual_type, actual_kind, + callee.arg_names[i], callee.arg_kinds[i]) + for actual_item in expanded_actuals: + check_arg(actual_item, actual_type, arg_kinds[actual], callee.arg_types[i], actual + 1, i + 1, callee, context, messages) - # There may be some remaining tuple varargs items that haven't - # been checked yet. Handle them. - tuplet = arg_types[actual] - if (callee.arg_kinds[i] == nodes.ARG_STAR and - arg_kinds[actual] == nodes.ARG_STAR and - isinstance(tuplet, TupleType)): - while mapper.tuple_index < len(tuplet.items): - actual_types = mapper.get_actual_type(arg_type, - arg_kinds[actual], - callee.arg_names[i]) - assert len(actual_types) == 1 - check_arg(actual_types[0], arg_type, arg_kinds[actual], - callee.arg_types[i], - actual + 1, i + 1, callee, context, messages) - def check_arg(self, caller_type: Type, original_caller_type: Type, caller_kind: int, callee_type: Type, n: int, m: int, callee: CallableType, @@ -1698,7 +1677,7 @@ def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, raise Finished try: - self.check_argument_types(arg_types, arg_kinds, callee.arg_names, callee, + self.check_argument_types(arg_types, arg_kinds, callee, formal_to_actual, context=context, check_arg=check_arg) return True except Finished: diff --git a/mypy/constraints.py b/mypy/constraints.py index 09fdade801ed..2c07750cf056 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -13,7 +13,7 @@ from mypy.sametypes import is_same_type from mypy.erasetype import erase_typevars from mypy.nodes import COVARIANT, CONTRAVARIANT -from mypy.argmap import ArgTypeMapper +from mypy.argmap import ArgTypeExpander MYPY = False if MYPY: @@ -54,7 +54,7 @@ def infer_constraints_for_callable( Return a list of constraints. """ constraints = [] # type: List[Constraint] - mapper = ArgTypeMapper() + mapper = ArgTypeExpander() for i, actuals in enumerate(formal_to_actual): for actual in actuals: @@ -62,8 +62,8 @@ def infer_constraints_for_callable( if actual_arg_type is None: continue - actual_types = mapper.get_actual_type(actual_arg_type, arg_kinds[actual], - callee.arg_names[i]) + actual_types = mapper.expand_actual_type(actual_arg_type, arg_kinds[actual], + callee.arg_names[i], callee.arg_kinds[i]) for actual_type in actual_types: c = infer_constraints(callee.arg_types[i], actual_type, SUPERTYPE_OF) From 568a02f545f2d5cd979cc6a9c676ceea8ac8d445 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Nov 2018 12:20:39 +0000 Subject: [PATCH 11/17] Add test case --- test-data/unit/check-varargs.test | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index 81701f8eea01..0cd0d1e6e308 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -551,6 +551,16 @@ main:10: error: Incompatible types in assignment (expression has type "List[]", variable has type "List[A]") main:11: error: Argument 1 to "f" of "G" has incompatible type "*List[A]"; expected "B" +[case testCallerTupleVarArgsAndGenerics] +# flags: --strict-optional +from typing import TypeVar + +T = TypeVar('T') + +def f(*args: T) -> T: ... +reveal_type(f(*(1, None))) # E: Revealed type is 'Union[builtins.int, None]' +[builtins fixtures/tuple.pyi] + -- Comment signatures -- ------------------ From 6bd0162a923bdd4ea01d9eb6b306d2c346c8c123 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Nov 2018 12:41:41 +0000 Subject: [PATCH 12/17] Fix multiple caller *args and refactor --- mypy/argmap.py | 42 +++++++++++++++---------------- mypy/checkexpr.py | 9 +++---- mypy/constraints.py | 10 +++----- test-data/unit/check-varargs.test | 2 ++ 4 files changed, 31 insertions(+), 32 deletions(-) diff --git a/mypy/argmap.py b/mypy/argmap.py index cf1323f005ce..493b646cd80f 100644 --- a/mypy/argmap.py +++ b/mypy/argmap.py @@ -117,10 +117,11 @@ class ArgTypeExpander: . def f(x: int, *args: str) -> None: ... . f(*(1, 'x', 1.1) - We'd call expand_actual_type twice: + We'd call expand_actual_type three times: - 1. The first call would provide 'int' as the type of 'x'. - 2. The second call would provide 'str' and 'float' as the types of '*args'. + 1. The first call would provide 'int' as the actual type of 'x' (from '1'). + 2. The second call would provide 'str' as one of the actual types for '*args'. + 2. The third call would provide 'float' as one of the actual types for '*args'. Construct a separate instance for each call since instances have per-call state. @@ -136,7 +137,7 @@ def expand_actual_type(self, actual_type: Type, actual_kind: int, formal_name: Optional[str], - formal_kind: int) -> List[Type]: + formal_kind: int) -> Type: """Return the actual (caller) type(s) of a formal argument with the given kinds. If the actual argument is a tuple *args, return the individual tuple item(s) that @@ -152,40 +153,39 @@ def expand_actual_type(self, if isinstance(actual_type, Instance): if actual_type.type.fullname() == 'builtins.list': # List *arg. - return [actual_type.args[0]] + return actual_type.args[0] elif actual_type.args: # TODO: Try to map type arguments to Iterable - return [actual_type.args[0]] + return actual_type.args[0] else: - return [AnyType(TypeOfAny.from_error)] + return AnyType(TypeOfAny.from_error) elif isinstance(actual_type, TupleType): # Get the next tuple item of a tuple *arg. - if formal_kind != nodes.ARG_STAR: - self.tuple_index += 1 - return [actual_type.items[self.tuple_index - 1]] + if self.tuple_index >= len(actual_type.items): + # Exhausted a tuple -- continue to the next *args. + self.tuple_index = 1 else: - # Callee takes *args. Give all remaining actual *args. - return actual_type.items[self.tuple_index:] + self.tuple_index += 1 + return actual_type.items[self.tuple_index - 1] else: - return [AnyType(TypeOfAny.from_error)] + return AnyType(TypeOfAny.from_error) elif actual_kind == nodes.ARG_STAR2: if isinstance(actual_type, TypedDictType): if formal_kind != nodes.ARG_STAR2 and formal_name in actual_type.items: # Lookup type based on keyword argument name. assert formal_name is not None - self.kwargs_used.add(formal_name) - return [actual_type.items[formal_name]] else: - # Callee takes **kwargs. Give all remaining actual **kwargs. - return [value for key, value in actual_type.items.items() - if key not in self.kwargs_used] + # Pick an arbitrary item if no specified keyword is expected. + formal_name = (set(actual_type.items.keys()) - self.kwargs_used).pop() + self.kwargs_used.add(formal_name) + return actual_type.items[formal_name] elif (isinstance(actual_type, Instance) and (actual_type.type.fullname() == 'builtins.dict')): # Dict **arg. # TODO: Handle arbitrary Mapping - return [actual_type.args[1]] + return actual_type.args[1] else: - return [AnyType(TypeOfAny.from_error)] + return AnyType(TypeOfAny.from_error) else: # No translation for other kinds -- 1:1 mapping. - return [actual_type] + return actual_type diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 0ad883d34e28..b7bb85fb8b55 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1193,13 +1193,12 @@ def check_argument_types(self, not self.is_valid_keyword_var_arg(actual_type)): is_mapping = is_subtype(actual_type, self.chk.named_type('typing.Mapping')) messages.invalid_keyword_var_arg(actual_type, is_mapping, context) - expanded_actuals = mapper.expand_actual_type( + expanded_actual = mapper.expand_actual_type( actual_type, actual_kind, callee.arg_names[i], callee.arg_kinds[i]) - for actual_item in expanded_actuals: - check_arg(actual_item, actual_type, arg_kinds[actual], - callee.arg_types[i], - actual + 1, i + 1, callee, context, messages) + check_arg(expanded_actual, actual_type, arg_kinds[actual], + callee.arg_types[i], + actual + 1, i + 1, callee, context, messages) def check_arg(self, caller_type: Type, original_caller_type: Type, caller_kind: int, diff --git a/mypy/constraints.py b/mypy/constraints.py index 2c07750cf056..f4530f6c79df 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -62,12 +62,10 @@ def infer_constraints_for_callable( if actual_arg_type is None: continue - actual_types = mapper.expand_actual_type(actual_arg_type, arg_kinds[actual], - callee.arg_names[i], callee.arg_kinds[i]) - for actual_type in actual_types: - c = infer_constraints(callee.arg_types[i], actual_type, - SUPERTYPE_OF) - constraints.extend(c) + actual_type = mapper.expand_actual_type(actual_arg_type, arg_kinds[actual], + callee.arg_names[i], callee.arg_kinds[i]) + c = infer_constraints(callee.arg_types[i], actual_type, SUPERTYPE_OF) + constraints.extend(c) return constraints diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index 0cd0d1e6e308..df036c04abf1 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -125,8 +125,10 @@ f(*it1, '') # E: Argument 2 to "f" has incompatible type "str"; expected "int" def f(*x: int) -> None: pass it1 = (1, 2) +it2 = ('',) f(*it1, 1, 2) f(*it1, 1, *it1, 2) +f(*it1, 1, *it2, 2) # E: Argument 3 to "f" has incompatible type "*Tuple[str]"; expected "int" f(*it1, '') # E: Argument 2 to "f" has incompatible type "str"; expected "int" [builtins fixtures/for.pyi] From f3a0fff3aab692f14da1e7168025b517a680b6a2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Nov 2018 12:56:20 +0000 Subject: [PATCH 13/17] Add test case with multiple caller **kwargs arguments --- test-data/unit/check-typeddict.test | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 74f24638bb7b..95bfd4f963b3 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -1467,3 +1467,27 @@ g(1, **a) # E: "g" gets multiple values for keyword argument "x" g(1, **b) # E: "g" gets multiple values for keyword argument "x" \ # E: Argument "x" to "g" has incompatible type "str"; expected "int" [builtins fixtures/dict.pyi] + +[case testTypedDictAsStarStarTwice] +from mypy_extensions import TypedDict + +A = TypedDict('A', {'x': int, 'y': str}) +B = TypedDict('B', {'z': bytes}) +C = TypedDict('C', {'x': str, 'z': bytes}) + +def f1(x: int, y: str, z: bytes) -> None: ... +def f2(x: int, y: float, z: bytes) -> None: ... +def f3(x: int, y: str, z: float) -> None: ... + +a: A +b: B +c: C +f1(**a, **b) +f1(**b, **a) +f2(**a, **b) # E: Argument "y" to "f2" has incompatible type "str"; expected "float" +f3(**a, **b) # E: Argument "z" to "f3" has incompatible type "bytes"; expected "float" +f3(**b, **a) # E: Argument "z" to "f3" has incompatible type "bytes"; expected "float" +f1(**a, **c) # E: "f1" gets multiple values for keyword argument "x" \ + # E: Argument "x" to "f1" has incompatible type "str"; expected "int" +f1(**c, **a) # E: "f1" gets multiple values for keyword argument "x" \ + # E: Argument "x" to "f1" has incompatible type "str"; expected "int" From 0a64b8c23f25ca0dd59cb98d5c684b173d25df0e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Nov 2018 13:00:53 +0000 Subject: [PATCH 14/17] Minor test case update --- test-data/unit/check-varargs.test | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index df036c04abf1..e4f3be15fc0d 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -553,7 +553,7 @@ main:10: error: Incompatible types in assignment (expression has type "List[]", variable has type "List[A]") main:11: error: Argument 1 to "f" of "G" has incompatible type "*List[A]"; expected "B" -[case testCallerTupleVarArgsAndGenerics] +[case testCallerTupleVarArgsAndGenericCalleeVarArg] # flags: --strict-optional from typing import TypeVar @@ -561,6 +561,8 @@ T = TypeVar('T') def f(*args: T) -> T: ... reveal_type(f(*(1, None))) # E: Revealed type is 'Union[builtins.int, None]' +reveal_type(f(1, *(None, 1))) # E: Revealed type is 'Union[builtins.int, None]' +reveal_type(f(1, *(1, None))) # E: Revealed type is 'Union[builtins.int, None]' [builtins fixtures/tuple.pyi] From 17b2a35fc478ae687d64919cc66ace5379ffa51b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Nov 2018 14:05:35 +0000 Subject: [PATCH 15/17] Fix lint --- mypy/argmap.py | 2 +- mypy/checkexpr.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/argmap.py b/mypy/argmap.py index 493b646cd80f..b3e0325e294e 100644 --- a/mypy/argmap.py +++ b/mypy/argmap.py @@ -180,7 +180,7 @@ def expand_actual_type(self, self.kwargs_used.add(formal_name) return actual_type.items[formal_name] elif (isinstance(actual_type, Instance) - and (actual_type.type.fullname() == 'builtins.dict')): + and (actual_type.type.fullname() == 'builtins.dict')): # Dict **arg. # TODO: Handle arbitrary Mapping return actual_type.args[1] diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index b7bb85fb8b55..5dc444d2140c 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1144,7 +1144,7 @@ def check_for_extra_actual_arguments(self, messages.unexpected_keyword_argument(callee, act_name, context) is_unexpected_arg_error = True elif ((kind == nodes.ARG_STAR and nodes.ARG_STAR not in callee.arg_kinds) - or kind == nodes.ARG_STAR2): + or kind == nodes.ARG_STAR2): actual_type = actual_types[i] if isinstance(actual_type, (TupleType, TypedDictType)): if all_actuals.count(i) < len(actual_type.items): From 113d6d99ad586e74ca19e0c1dfe9da8c1abef894 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Nov 2018 19:30:51 +0000 Subject: [PATCH 16/17] Update based on feedback --- mypy/argmap.py | 18 +++++++++--------- mypy/checkexpr.py | 4 ++-- test-data/unit/check-varargs.test | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/mypy/argmap.py b/mypy/argmap.py index b3e0325e294e..2540d8480d63 100644 --- a/mypy/argmap.py +++ b/mypy/argmap.py @@ -73,7 +73,7 @@ def map_actuals_to_formals(caller_kinds: List[int], elif nodes.ARG_STAR2 in callee_kinds: map[callee_kinds.index(nodes.ARG_STAR2)].append(i) else: - # We don't exactly which **kwargs are provided by the + # We don't exactly know which **kwargs are provided by the # caller. Assume that they will fill the remaining arguments. for j in range(ncallee): # TODO tuple varargs complicate this @@ -114,8 +114,8 @@ class ArgTypeExpander: Example: - . def f(x: int, *args: str) -> None: ... - . f(*(1, 'x', 1.1) + def f(x: int, *args: str) -> None: ... + f(*(1, 'x', 1.1)) We'd call expand_actual_type three times: @@ -123,8 +123,8 @@ class ArgTypeExpander: 2. The second call would provide 'str' as one of the actual types for '*args'. 2. The third call would provide 'float' as one of the actual types for '*args'. - Construct a separate instance for each call since instances have per-call - state. + A single instance can process all the arguments for a single call. Each call + needs a separate instance since instances have per-call state. """ def __init__(self) -> None: @@ -140,11 +140,11 @@ def expand_actual_type(self, formal_kind: int) -> Type: """Return the actual (caller) type(s) of a formal argument with the given kinds. - If the actual argument is a tuple *args, return the individual tuple item(s) that - map(s) to the formal arg. + If the actual argument is a tuple *args, return the next individual tuple item that + maps to the formal arg. - If the actual argument is a TypedDict **kwargs, return the matching typed dict dict - value type(s) based on formal argument name and kind. + If the actual argument is a TypedDict **kwargs, return the next matching typed dict + value type based on formal argument name and kind. This is supposed to be called for each formal, in order. Call multiple times per formal if multiple actuals map to a formal. diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 5dc444d2140c..45e302f6595b 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1064,7 +1064,7 @@ def check_argument_count(self, Return False if there were any errors. Otherwise return True """ if messages: - assert context, "Internal error: messages gives without context" + assert context, "Internal error: messages given without context" elif context is None: context = TempNode(AnyType(TypeOfAny.special_form)) # Avoid "is None" checks @@ -1121,7 +1121,7 @@ def check_for_extra_actual_arguments(self, """Check for extra actual arguments. Return tuple (was everything ok, - was there an extra keyword argument error). + was there an extra keyword argument error [used to avoid duplicate errors]). """ is_unexpected_arg_error = False # Keep track of errors to avoid duplicate errors diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index e4f3be15fc0d..75d24fad0fb8 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -111,6 +111,20 @@ f(*it1) f(*it2) # E: Argument 1 to "f" has incompatible type "*Iterable[str]"; expected "int" [builtins fixtures/for.pyi] +[case testCallVarargsFunctionWithTwoTupleStarArgs] +from typing import TypeVar, Tuple + +T1 = TypeVar('T1') +T2 = TypeVar('T2') +T3 = TypeVar('T3') +T4 = TypeVar('T4') + +def f(a: T1, b: T2, c: T3, d: T4) -> Tuple[T1, T2, T3, T4]: ... +x: Tuple[int, str] +y: Tuple[float, bool] +reveal_type(f(*x, *y)) # E: Revealed type is 'Tuple[builtins.int*, builtins.str*, builtins.float*, builtins.bool*]' +[builtins fixtures/list.pyi] + [case testCallVarargsFunctionWithIterableAndPositional] from typing import Iterable From 05e49e53fb372863edb8ba37a08c1742a76cb0b2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Nov 2018 19:35:59 +0000 Subject: [PATCH 17/17] Comment update --- mypy/argmap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/argmap.py b/mypy/argmap.py index 2540d8480d63..788a0c744da2 100644 --- a/mypy/argmap.py +++ b/mypy/argmap.py @@ -76,7 +76,8 @@ def map_actuals_to_formals(caller_kinds: List[int], # We don't exactly know which **kwargs are provided by the # caller. Assume that they will fill the remaining arguments. for j in range(ncallee): - # TODO tuple varargs complicate this + # TODO: If there are also tuple varargs, we might be missing some potential + # matches if the tuple was short enough to not match everything. no_certain_match = ( not map[j] or caller_kinds[map[j][0]] == nodes.ARG_STAR) if ((callee_names[j] and no_certain_match)