Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve usage of outer context for inference #5699

Merged
merged 14 commits into from
Oct 2, 2018
Merged

Improve usage of outer context for inference #5699

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
decb5fd
Don't show errors before inference is done
Sep 29, 2018
a262692
Tweak tests
ilevkivskyi Sep 29, 2018
fb841b7
Remove some tests
ilevkivskyi Sep 29, 2018
d941c23
Update a comment in skipped test and remove now unneded fixture updates
ilevkivskyi Sep 29, 2018
e18a521
Add special case for optional return vs optional context
ilevkivskyi Sep 30, 2018
517d7c2
Re-order special cases
ilevkivskyi Sep 30, 2018
fc603f4
Fix long line
ilevkivskyi Sep 30, 2018
826806c
Updates on feedback; will add tests later
ilevkivskyi Sep 30, 2018
17146f7
Add more tests
Oct 1, 2018
3441ed5
Merge branch 'outer-context' of https://github.com/ilevkivskyi/mypy i…
Oct 1, 2018
9a0af3c
Address CR
Oct 1, 2018
31d1ca2
Merge branch 'master' into outer-context
ilevkivskyi Oct 1, 2018
f3a28c2
Fix broken merge
ilevkivskyi Oct 1, 2018
3cd7d25
Switch to invariant instances in special case
Oct 1, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 30 additions & 8 deletions mypy/applytype.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,17 @@


def apply_generic_arguments(callable: CallableType, orig_types: Sequence[Optional[Type]],
msg: MessageBuilder, context: Context) -> CallableType:
msg: MessageBuilder, context: Context,
skip_unsatisfied: bool = False) -> CallableType:
"""Apply generic type arguments to a callable type.

For example, applying [int] to 'def [T] (T) -> T' results in
'def (int) -> int'.

Note that each type can be None; in this case, it will not be applied.

If `skip_unsatisfied` is True, only apply those types that satisfy type variable
bound or constraints (and replace the type with `None`), instead of giving an error.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this docstring correct? It seems to me that the type is not replaced at all if it's unsatisfied.

"""
tvars = callable.variables
assert len(tvars) == len(orig_types)
Expand All @@ -25,7 +29,9 @@ def apply_generic_arguments(callable: CallableType, orig_types: Sequence[Optiona
for i, type in enumerate(types):
assert not isinstance(type, PartialType), "Internal error: must never apply partial type"
values = callable.variables[i].values
if values and type:
if type is None:
continue
if values:
if isinstance(type, AnyType):
continue
if isinstance(type, TypeVarType) and type.values:
Expand All @@ -34,15 +40,31 @@ def apply_generic_arguments(callable: CallableType, orig_types: Sequence[Optiona
if all(any(is_same_type(v, v1) for v in values)
for v1 in type.values):
continue
matching = []
for value in values:
if mypy.subtypes.is_subtype(type, value):
types[i] = value
break
matching.append(value)
if matching:
best = matching[0]
# If there are more than one matching value, we select the narrowest
for match in matching[1:]:
if mypy.subtypes.is_subtype(match, best):
best = match
types[i] = best
else:
msg.incompatible_typevar_value(callable, type, callable.variables[i].name, context)
upper_bound = callable.variables[i].upper_bound
if type and not mypy.subtypes.is_subtype(type, upper_bound):
msg.incompatible_typevar_value(callable, type, callable.variables[i].name, context)
if skip_unsatisfied:
types[i] = None
else:
msg.incompatible_typevar_value(callable, type, callable.variables[i].name,
context)
else:
upper_bound = callable.variables[i].upper_bound
if not mypy.subtypes.is_subtype(type, upper_bound):
if skip_unsatisfied:
types[i] = None
else:
msg.incompatible_typevar_value(callable, type, callable.variables[i].name,
context)

# Create a map from type variable id to target type.
id_to_type = {} # type: Dict[TypeVarId, Type]
Expand Down
54 changes: 37 additions & 17 deletions mypy/checkexpr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -823,20 +823,36 @@ def infer_function_type_arguments_using_context(
# valid results.
erased_ctx = replace_meta_vars(ctx, ErasedType())
ret_type = callable.ret_type
if isinstance(ret_type, TypeVarType):
if ret_type.values or (not isinstance(ctx, Instance) or
not ctx.args):
# The return type is a type variable. If it has values, we can't easily restrict
# type inference to conform to the valid values. If it's unrestricted, we could
# infer a too general type for the type variable if we use context, and this could
# result in confusing and spurious type errors elsewhere.
#
# Give up and just use function arguments for type inference. As an exception,
# if the context is a generic instance type, actually use it as context, as
# this *seems* to usually be the reasonable thing to do.
#
# See also github issues #462 and #360.
ret_type = NoneTyp()
if mypy.checker.is_optional(ret_type) and mypy.checker.is_optional(ctx):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now that is_optional is used from two modules, I'd move it to somewhere else, such as mypy.types. remove_optional is similar.

# If both the context and the return type are optional, unwrap the optional,
# since in 99% cases this is what a user expects. In other words, we replace
# Optional[T] <: Optional[int]
# with
# T <: int
# while the former would infer T <: Optional[int].
ret_type = mypy.checker.remove_optional(ret_type)
erased_ctx = mypy.checker.remove_optional(erased_ctx)
#
# TODO: Instead of this hack and the one below, we need to use outer and
# inner contexts at the same time. This is however not easy because of two
# reasons:
# * We need to support constraints like [1 <: 2, 2 <: X], i.e. with variables
# on both sides. (This is not too hard.)
# * We need to update all the inference "infrastructure", so that all
# variables in an expression are inferred at the same time.
# (And this is hard, also we need to be careful with lambdas that require
# two passes.)
if isinstance(ret_type, TypeVarType) and not (isinstance(ctx, Instance) and ctx.args):
# Another special case: the return type is a type variable. If it's unrestricted,
# we could infer a too general type for the type variable if we use context,
# and this could result in confusing and spurious type errors elsewhere.
#
# Give up and just use function arguments for type inference. As an exception,
# if the context is a generic instance type, actually use it as context, as
# this *seems* to usually be the reasonable thing to do.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe this is only reasonable if the generic instance type is invariant in some type arg? I wonder if that would be a bit more "principled".

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TBH I am not sure this is better. Currently, we ignore the return context for cases like T <: int but use it for T <: Iterable[int]. So the only way how only using T <: List[int] will be better, is because invariant containers statistically have less subtypes. As I understand the original problem this solves is like this:

def f(x: List[T]) -> T: ...
y: object = f([1, 2, 3])  # Incompatible argument type "List[int]", expected "List[object]"

In other words the problem appears when the return type appears as a variable in invariant context in one of argument types.

However, if you prefer to only use invariant ones, I can update this.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What I meant is that if the context is, say, List[object], List[int] wouldn't be valid due to invariance, so we are more likely to want to use the surrounding type context and not rely on arguments only. However, if the context is Iterable[object], Iterable[int] would be okay, due co covariance, so we ignore the context. Whether there are many subtypes or not doesn't seem significant?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, this is actually exactly what I meant :-) Iterable[object] has more subtypes, such as Iterable[int]. While List[object] doesn't have such, so it is typically better to still use it as a context. OK, I will modify the special case to reflect this.

#
# See also github issues #462 and #360.
return callable.copy_modified()
args = infer_type_arguments(callable.type_var_ids(), ret_type, erased_ctx)
# Only substitute non-Uninhabited and non-erased types.
new_args = [] # type: List[Optional[Type]]
Expand All @@ -845,7 +861,10 @@ def infer_function_type_arguments_using_context(
new_args.append(None)
else:
new_args.append(arg)
return self.apply_generic_arguments(callable, new_args, error_context)
# Don't show errors after we have only used the outer context for inference.
# We will use argument context to infer more variables.
return self.apply_generic_arguments(callable, new_args, error_context,
skip_unsatisfied=True)

def infer_function_type_arguments(self, callee_type: CallableType,
args: List[Expression],
Expand Down Expand Up @@ -1613,9 +1632,10 @@ def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int,
return False

def apply_generic_arguments(self, callable: CallableType, types: Sequence[Optional[Type]],
context: Context) -> CallableType:
context: Context, skip_unsatisfied: bool = False) -> CallableType:
"""Simple wrapper around mypy.applytype.apply_generic_arguments."""
return applytype.apply_generic_arguments(callable, types, self.msg, context)
return applytype.apply_generic_arguments(callable, types, self.msg, context,
skip_unsatisfied=skip_unsatisfied)

def visit_member_expr(self, e: MemberExpr, is_lvalue: bool = False) -> Type:
"""Visit member expression (of form e.id)."""
Expand Down
2 changes: 1 addition & 1 deletion test-data/unit/check-generics.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -849,7 +849,7 @@ def fun2(v: Vec[T], scale: T) -> Vec[T]:
return v

reveal_type(fun1([(1, 1)])) # E: Revealed type is 'builtins.int*'
fun1(1) # E: Argument 1 to "fun1" has incompatible type "int"; expected "List[Tuple[int, int]]"
fun1(1) # E: Argument 1 to "fun1" has incompatible type "int"; expected "List[Tuple[bool, bool]]"
fun1([(1, 'x')]) # E: Cannot infer type argument 1 of "fun1"

reveal_type(fun2([(1, 1)], 1)) # E: Revealed type is 'builtins.list[Tuple[builtins.int*, builtins.int*]]'
Expand Down
Loading