Improve join and meet of callables and overloads (python#2833)

Fixes python#1983

Here I implement:

* join(Callable[[A1], R1]), Callable[[A2], R2]) == Callable[[meet(A1, A2)], join(R1, R2)]
* meet(Callable[[A1], R1]), Callable[[A2], R2]) == Callable[[join(A1, A2)], meet(R1, R2)]

plus special cases for Any, overloads, and callable type objects.

The meet and join are still not perfect, but I think this PR improves the situation.

mypy/checkexpr.py

@@ -2435,9 +2435,12 @@ def overload_arg_similarity(actual: Type, formal: Type) -> int:
             (isinstance(actual, Instance) and actual.type.fallback_to_any)):
         # These could match anything at runtime.
         return 2
-    if isinstance(formal, CallableType) and isinstance(actual, (CallableType, Overloaded)):
-        # TODO: do more sophisticated callable matching
-        return 2
+    if isinstance(formal, CallableType):
+        if isinstance(actual, (CallableType, Overloaded)):
+            # TODO: do more sophisticated callable matching
+            return 2
+        if isinstance(actual, TypeType):
+            return 2 if is_subtype(actual, formal) else 0
     if isinstance(actual, NoneTyp):
         if not experiments.STRICT_OPTIONAL:
             # NoneTyp matches anything if we're not doing strict Optional checking

mypy/join.py

@@ -152,9 +152,14 @@ def visit_instance(self, t: Instance) -> Type:
             return self.default(self.s)
 
     def visit_callable_type(self, t: CallableType) -> Type:
-        # TODO: Consider subtyping instead of just similarity.
         if isinstance(self.s, CallableType) and is_similar_callables(t, self.s):
-            return combine_similar_callables(t, self.s)
+            if is_equivalent(t, self.s):
+                return combine_similar_callables(t, self.s)
+            result = join_similar_callables(t, self.s)
+            if any(isinstance(tp, (NoneTyp, UninhabitedType)) for tp in result.arg_types):
+                # We don't want to return unusable Callable, attempt fallback instead.
+                return join_types(t.fallback, self.s)
+            return result
         elif isinstance(self.s, Overloaded):
             # Switch the order of arguments to that we'll get to visit_overloaded.
             return join_types(t, self.s)
@@ -189,15 +194,18 @@ def visit_overloaded(self, t: Overloaded) -> Type:
         #   join(Ov([int, Any] -> Any, [str, Any] -> Any), [Any, int] -> Any) ==
         #       Ov([Any, int] -> Any, [Any, int] -> Any)
         #
-        # TODO: Use callable subtyping instead of just similarity.
+        # TODO: Consider more cases of callable subtyping.
         result = []  # type: List[CallableType]
         s = self.s
         if isinstance(s, FunctionLike):
             # The interesting case where both types are function types.
             for t_item in t.items():
                 for s_item in s.items():
                     if is_similar_callables(t_item, s_item):
-                        result.append(combine_similar_callables(t_item, s_item))
+                        if is_equivalent(t_item, s_item):
+                            result.append(combine_similar_callables(t_item, s_item))
+                        elif is_subtype(t_item, s_item):
+                            result.append(s_item)
             if result:
                 # TODO: Simplify redundancies from the result.
                 if len(result) == 1:
@@ -323,12 +331,30 @@ def is_better(t: Type, s: Type) -> bool:
 
 
 def is_similar_callables(t: CallableType, s: CallableType) -> bool:
-    """Return True if t and s are equivalent and have identical numbers of
+    """Return True if t and s have identical numbers of
     arguments, default arguments and varargs.
     """
 
-    return (len(t.arg_types) == len(s.arg_types) and t.min_args == s.min_args
-            and t.is_var_arg == s.is_var_arg and is_equivalent(t, s))
+    return (len(t.arg_types) == len(s.arg_types) and t.min_args == s.min_args and
+            t.is_var_arg == s.is_var_arg)
+
+
+def join_similar_callables(t: CallableType, s: CallableType) -> CallableType:
+    from mypy.meet import meet_types
+    arg_types = []  # type: List[Type]
+    for i in range(len(t.arg_types)):
+        arg_types.append(meet_types(t.arg_types[i], s.arg_types[i]))
+    # TODO in combine_similar_callables also applies here (names and kinds)
+    # The fallback type can be either 'function' or 'type'. The result should have 'type' as
+    # fallback only if both operands have it as 'type'.
+    if t.fallback.type.fullname() != 'builtins.type':
+        fallback = t.fallback
+    else:
+        fallback = s.fallback
+    return t.copy_modified(arg_types=arg_types,
+                           ret_type=join_types(t.ret_type, s.ret_type),
+                           fallback=fallback,
+                           name=None)
 
 
 def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType:

mypy/meet.py

@@ -220,7 +220,13 @@ def visit_instance(self, t: Instance) -> Type:
 
     def visit_callable_type(self, t: CallableType) -> Type:
         if isinstance(self.s, CallableType) and is_similar_callables(t, self.s):
-            return combine_similar_callables(t, self.s)
+            if is_equivalent(t, self.s):
+                return combine_similar_callables(t, self.s)
+            result = meet_similar_callables(t, self.s)
+            if isinstance(result.ret_type, UninhabitedType):
+                # Return a plain None or <uninhabited> instead of a weird function.
+                return self.default(self.s)
+            return result
         else:
             return self.default(self.s)
 
@@ -279,3 +285,21 @@ def default(self, typ: Type) -> Type:
                 return UninhabitedType()
             else:
                 return NoneTyp()
+
+
+def meet_similar_callables(t: CallableType, s: CallableType) -> CallableType:
+    from mypy.join import join_types
+    arg_types = []  # type: List[Type]
+    for i in range(len(t.arg_types)):
+        arg_types.append(join_types(t.arg_types[i], s.arg_types[i]))
+    # TODO in combine_similar_callables also applies here (names and kinds)
+    # The fallback type can be either 'function' or 'type'. The result should have 'function' as
+    # fallback only if both operands have it as 'function'.
+    if t.fallback.type.fullname() != 'builtins.function':
+        fallback = t.fallback
+    else:
+        fallback = s.fallback
+    return t.copy_modified(arg_types=arg_types,
+                           ret_type=meet_types(t.ret_type, s.ret_type),
+                           fallback=fallback,
+                           name=None)

mypy/test/testtypes.py

@@ -392,13 +392,16 @@ def test_function_types(self) -> None:
 
         self.assert_join(self.callable(self.fx.a, self.fx.b),
                          self.callable(self.fx.b, self.fx.b),
-                         self.fx.function)
+                         self.callable(self.fx.b, self.fx.b))
         self.assert_join(self.callable(self.fx.a, self.fx.b),
                          self.callable(self.fx.a, self.fx.a),
-                         self.fx.function)
+                         self.callable(self.fx.a, self.fx.a))
         self.assert_join(self.callable(self.fx.a, self.fx.b),
                          self.fx.function,
                          self.fx.function)
+        self.assert_join(self.callable(self.fx.a, self.fx.b),
+                         self.callable(self.fx.d, self.fx.b),
+                         self.fx.function)
 
     def test_type_vars(self) -> None:
         self.assert_join(self.fx.t, self.fx.t, self.fx.t)
@@ -560,13 +563,14 @@ def test_generic_interfaces(self) -> None:
     def test_simple_type_objects(self) -> None:
         t1 = self.type_callable(self.fx.a, self.fx.a)
         t2 = self.type_callable(self.fx.b, self.fx.b)
+        tr = self.type_callable(self.fx.b, self.fx.a)
 
         self.assert_join(t1, t1, t1)
         j = join_types(t1, t1)
         assert isinstance(j, CallableType)
         assert_true(j.is_type_obj())
 
-        self.assert_join(t1, t2, self.fx.type_type)
+        self.assert_join(t1, t2, tr)
         self.assert_join(t1, self.fx.type_type, self.fx.type_type)
         self.assert_join(self.fx.type_type, self.fx.type_type,
                          self.fx.type_type)
@@ -658,10 +662,10 @@ def test_function_types(self) -> None:
 
         self.assert_meet(self.callable(self.fx.a, self.fx.b),
                          self.callable(self.fx.b, self.fx.b),
-                         NoneTyp())
+                         self.callable(self.fx.a, self.fx.b))
         self.assert_meet(self.callable(self.fx.a, self.fx.b),
                          self.callable(self.fx.a, self.fx.a),
-                         NoneTyp())
+                         self.callable(self.fx.a, self.fx.b))
 
     def test_type_vars(self) -> None:
         self.assert_meet(self.fx.t, self.fx.t, self.fx.t)

test-data/unit/check-inference.test

@@ -696,6 +696,53 @@ i(b, a, b)
 i(a, b, b) # E: Argument 1 to "i" has incompatible type List[int]; expected List[str]
 [builtins fixtures/list.pyi]
 
+[case testCallableListJoinInference]
+from typing import Any, Callable
+
+def fun() -> None:
+    callbacks = [
+        callback1,
+        callback2,
+    ]
+
+    for c in callbacks:
+        call(c, 1234) # this must not fail
+
+def callback1(i: int) -> int:
+    return i
+def callback2(i: int) -> str:
+    return 'hello'
+def call(c: Callable[[int], Any], i: int) -> None:
+    c(i)
+[builtins fixtures/list.pyi]
+[out]
+
+[case testCallableMeetAndJoin]
+# flags: --python-version 3.6
+from typing import Callable, Any, TypeVar
+
+class A: ...
+class B(A): ...
+
+def f(c: Callable[[B], int]) -> None: ...
+
+c: Callable[[A], int]
+d: Callable[[B], int]
+
+lst = [c, d]
+reveal_type(lst) # E: Revealed type is 'builtins.list[def (__main__.B) -> builtins.int]'
+
+T = TypeVar('T')
+def meet_test(x: Callable[[T], int], y: Callable[[T], int]) -> T: ...
+
+CA = Callable[[A], A]
+CB = Callable[[B], B]
+
+ca: Callable[[CA], int]
+cb: Callable[[CB], int]
+reveal_type(meet_test(ca, cb)) # E: Revealed type is 'def (__main__.A) -> __main__.B'
+[builtins fixtures/list.pyi]
+[out]
 
 [case testUnionInferenceWithTypeVarValues]
 from typing import TypeVar, Union