From 437509c37d50d52ee4d7651013f7c555e43087c0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 7 Jun 2017 17:13:16 +0100 Subject: [PATCH 01/10] Support additional plugin hooks Includes these features: * Add hook that overrides the inferred type of an instance attribute. In particular, this can be used to override the type of `__call__`. * Add hook for custom semantic analysis of types. This makes it possible to have some limited custom syntax for generic types. (We can extend the syntactic possibilities in the future.) * Use the name of a callable to decide which hook to apply. This makes it possible to use custom hooks for callables returned by functions. --- mypy/build.py | 16 +++-- mypy/checkexpr.py | 2 + mypy/checkmember.py | 27 +++++-- mypy/plugin.py | 56 +++++++++++++-- mypy/semanal.py | 13 +++- mypy/test/testgraph.py | 3 + mypy/typeanal.py | 90 +++++++++++++++--------- mypy/types.py | 2 +- test-data/unit/check-custom-plugin.test | 52 ++++++++++++++ test-data/unit/plugins/attrhook.py | 21 ++++++ test-data/unit/plugins/named_callable.py | 28 ++++++++ test-data/unit/plugins/type_anal_hook.py | 38 ++++++++++ 12 files changed, 292 insertions(+), 56 deletions(-) create mode 100644 test-data/unit/plugins/attrhook.py create mode 100644 test-data/unit/plugins/named_callable.py create mode 100644 test-data/unit/plugins/type_anal_hook.py diff --git a/mypy/build.py b/mypy/build.py index e4b202c4e72b..5e40af8ebe81 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -171,8 +171,9 @@ def build(sources: List[BuildSource], lib_path.insert(0, alt_lib_path) reports = Reports(data_dir, options.report_dirs) - source_set = BuildSourceSet(sources) + errors = Errors(options.show_error_context, options.show_column_numbers) + plugin = load_custom_plugins(DefaultPlugin(options.python_version), options, errors) # Construct a build manager object to hold state during the build. # @@ -183,9 +184,8 @@ def build(sources: List[BuildSource], reports=reports, options=options, version_id=__version__, - plugin=DefaultPlugin(options.python_version)) - - manager.plugin = load_custom_plugins(manager.plugin, options, manager.errors) + plugin=plugin, + errors=errors) try: graph = dispatch(sources, manager) @@ -430,10 +430,11 @@ def __init__(self, data_dir: str, reports: Reports, options: Options, version_id: str, - plugin: Plugin) -> None: + plugin: Plugin, + errors: Errors) -> None: self.start_time = time.time() self.data_dir = data_dir - self.errors = Errors(options.show_error_context, options.show_column_numbers) + self.errors = errors self.errors.set_ignore_prefix(ignore_prefix) self.lib_path = tuple(lib_path) self.source_set = source_set @@ -442,8 +443,9 @@ def __init__(self, data_dir: str, self.version_id = version_id self.modules = {} # type: Dict[str, MypyFile] self.missing_modules = set() # type: Set[str] + self.plugin = plugin self.semantic_analyzer = SemanticAnalyzer(self.modules, self.missing_modules, - lib_path, self.errors) + lib_path, self.errors, self.plugin) self.modules = self.semantic_analyzer.modules self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) self.all_types = {} # type: Dict[Expression, Type] diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 027c0d85a854..e291a6eb3a9a 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -475,6 +475,8 @@ def check_call(self, callee: Type, args: List[Expression], """ arg_messages = arg_messages or self.msg if isinstance(callee, CallableType): + if callable_name is None and callee.name: + callable_name = callee.name if (isinstance(callable_node, RefExpr) and callable_node.fullname in ('enum.Enum', 'enum.IntEnum', 'enum.Flag', 'enum.IntFlag')): diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 66949144eed8..8b4fec1e9f7f 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -17,6 +17,7 @@ from mypy.expandtype import expand_type_by_instance, expand_type, freshen_function_type_vars from mypy.infer import infer_type_arguments from mypy.typevars import fill_typevars +from mypy.plugin import Plugin from mypy import messages from mypy import subtypes MYPY = False @@ -76,8 +77,9 @@ def analyze_member_access(name: str, if method.is_property: assert isinstance(method, OverloadedFuncDef) first_item = cast(Decorator, method.items[0]) + plugin = chk.plugin if chk is not None else None return analyze_var(name, first_item.var, typ, info, node, is_lvalue, msg, - original_type, not_ready_callback) + original_type, not_ready_callback, plugin) if is_lvalue: msg.cant_assign_to_method(node) signature = function_type(method, builtin_type('builtins.function')) @@ -227,8 +229,9 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo, v = vv.var if isinstance(v, Var): + plugin = chk.plugin if chk is not None else None return analyze_var(name, v, itype, info, node, is_lvalue, msg, - original_type, not_ready_callback) + original_type, not_ready_callback, plugin) elif isinstance(v, FuncDef): assert False, "Did not expect a function" elif not v and name not in ['__getattr__', '__setattr__', '__getattribute__']: @@ -270,7 +273,8 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo, def analyze_var(name: str, var: Var, itype: Instance, info: TypeInfo, node: Context, is_lvalue: bool, msg: MessageBuilder, original_type: Type, - not_ready_callback: Callable[[str, Context], None]) -> Type: + not_ready_callback: Callable[[str, Context], None], + plugin: Optional[Plugin]) -> Type: """Analyze access to an attribute via a Var node. This is conceptually part of analyze_member_access and the arguments are similar. @@ -308,15 +312,24 @@ def analyze_var(name: str, var: Var, itype: Instance, info: TypeInfo, node: Cont # A property cannot have an overloaded type => the cast # is fine. assert isinstance(signature, CallableType) - return signature.ret_type + result = signature.ret_type else: - return signature - return t + result = signature + else: + result = t + else: + result = t else: if not var.is_ready: not_ready_callback(var.name(), node) # Implicit 'Any' type. - return AnyType() + result = AnyType() + if plugin: + fullname = '{}.{}'.format(var.info.fullname(), name) + hook = plugin.get_attribute_hook(fullname) + if hook: + result = hook(original_type, result) + return result def freeze_type_vars(member_type: Type) -> None: diff --git a/mypy/plugin.py b/mypy/plugin.py index 7acd4d0b29a5..20a0290c5422 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -3,7 +3,7 @@ from mypy.nodes import Expression, StrExpr, IntExpr, UnaryExpr, Context from mypy.types import ( Type, Instance, CallableType, TypedDictType, UnionType, NoneTyp, FunctionLike, TypeVarType, - AnyType + AnyType, TypeList, UnboundType ) from mypy.messages import MessageBuilder @@ -11,12 +11,31 @@ # Create an Instance given full name of class and type arguments. NamedInstanceCallback = Callable[[str, List[Type]], Type] +AnalyzeArgListCallback = Callable[[TypeList], Optional[Tuple[List[Type], + List[int], + List[Optional[str]]]]] + # Some objects and callbacks that plugins can use to get information from the # type checker or to report errors. -PluginContext = NamedTuple('PluginContext', [('named_instance', NamedInstanceCallback), - ('msg', MessageBuilder), - ('context', Context)]) +PluginContext = NamedTuple( + 'PluginContext', + [ + ('named_instance', NamedInstanceCallback), + ('msg', MessageBuilder), + ('context', Context) + ] +) +SemanticAnalysisPluginContext = NamedTuple( + 'SemanticAnalysisPluginContext', + [ + ('named_instance', NamedInstanceCallback), + ('fail', Callable[[str, Context], None]), + ('analyze_type', Callable[[Type], Type]), + ('analyze_arg_list', AnalyzeArgListCallback), + ('context', Context) + ] +) # A callback that infers the return type of a function with a special signature. # @@ -58,6 +77,23 @@ Type # Return type inferred by the callback ] +AttributeHook = Callable[ + [ + Type, # Base object type + Type # Inferred attribute type + # TODO: Some context object? + ], + Type +] + +TypeAnalyzeHook = Callable[ + [ + UnboundType, + SemanticAnalysisPluginContext + ], + Type +] + class Plugin: """Base class of all type checker plugins. @@ -81,6 +117,12 @@ def get_method_signature_hook(self, fullname: str) -> Optional[MethodSignatureHo def get_method_hook(self, fullname: str) -> Optional[MethodHook]: return None + def get_attribute_hook(self, fullname: str) -> Optional[AttributeHook]: + return None + + def get_type_analyze_hook(self, fullname: str) -> Optional[TypeAnalyzeHook]: + return None + # TODO: metaclass / class decorator hook @@ -116,6 +158,12 @@ def get_method_signature_hook(self, fullname: str) -> Optional[MethodSignatureHo def get_method_hook(self, fullname: str) -> Optional[MethodHook]: return self._find_hook(lambda plugin: plugin.get_method_hook(fullname)) + def get_attribute_hook(self, fullname: str) -> Optional[AttributeHook]: + return self._find_hook(lambda plugin: plugin.get_attribute_hook(fullname)) + + def get_type_analyze_hook(self, fullname: str) -> Optional[TypeAnalyzeHook]: + return self._find_hook(lambda plugin: plugin.get_type_analyze_hook(fullname)) + def _find_hook(self, lookup: Callable[[Plugin], T]) -> Optional[T]: for plugin in self._plugins: hook = lookup(plugin) diff --git a/mypy/semanal.py b/mypy/semanal.py index 523edc8563e0..111bb3b498c2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -90,6 +90,7 @@ from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError from mypy.sametypes import is_same_type from mypy.options import Options +from mypy.plugin import Plugin from mypy import join @@ -230,11 +231,13 @@ class SemanticAnalyzer(NodeVisitor): is_stub_file = False # Are we analyzing a stub file? imports = None # type: Set[str] # Imported modules (during phase 2 analysis) errors = None # type: Errors # Keeps track of generated errors + plugin = None # type: Plugin # Mypy plugin for special casing of library features def __init__(self, modules: Dict[str, MypyFile], missing_modules: Set[str], - lib_path: List[str], errors: Errors) -> None: + lib_path: List[str], errors: Errors, + plugin: Plugin) -> None: """Construct semantic analyzer. Use lib_path to search for modules, and report analysis errors @@ -256,6 +259,7 @@ def __init__(self, self.postpone_nested_functions_stack = [FUNCTION_BOTH_PHASES] self.postponed_functions_stack = [] self.all_exports = set() # type: Set[str] + self.plugin = plugin def visit_file(self, file_node: MypyFile, fnam: str, options: Options) -> None: self.options = options @@ -1497,6 +1501,7 @@ def type_analyzer(self, *, self.lookup_fully_qualified, tvar_scope, self.fail, + self.plugin, aliasing=aliasing, allow_tuple_literal=allow_tuple_literal, allow_unnormalized=self.is_stub_file) @@ -1534,7 +1539,8 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.lookup_qualified, self.lookup_fully_qualified, self.tvar_scope, - self.fail, allow_unnormalized=True) + self.fail, + self.plugin, allow_unnormalized=True) if res and (not isinstance(res, Instance) or res.args): # TODO: What if this gets reassigned? name = s.lvalues[0] @@ -3099,7 +3105,8 @@ def visit_index_expr(self, expr: IndexExpr) -> None: self.lookup_qualified, self.lookup_fully_qualified, self.tvar_scope, - self.fail, allow_unnormalized=self.is_stub_file) + self.fail, + self.plugin, allow_unnormalized=self.is_stub_file) expr.analyzed = TypeAliasExpr(res, fallback=self.alias_fallback(res), in_runtime=True) elif refers_to_class_or_function(expr.base): diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py index d168ad53e236..58c4e8b3a9f6 100644 --- a/mypy/test/testgraph.py +++ b/mypy/test/testgraph.py @@ -10,6 +10,7 @@ from mypy.report import Reports from mypy.plugin import Plugin from mypy import defaults +from mypy.errors import Errors class GraphSuite(Suite): @@ -36,6 +37,7 @@ def test_scc(self) -> None: frozenset({'D'})}) def _make_manager(self) -> BuildManager: + errors = Errors() manager = BuildManager( data_dir='', lib_path=[], @@ -45,6 +47,7 @@ def _make_manager(self) -> BuildManager: options=Options(), version_id=__version__, plugin=Plugin(defaults.PYTHON3_VERSION), + errors=errors, ) return manager diff --git a/mypy/typeanal.py b/mypy/typeanal.py index d36ecb98de6a..9ec50a764fe4 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -24,6 +24,7 @@ from mypy.sametypes import is_same_type from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError from mypy.subtypes import is_subtype +from mypy.plugin import Plugin, SemanticAnalysisPluginContext from mypy import nodes from mypy import experiments @@ -54,6 +55,7 @@ def analyze_type_alias(node: Expression, lookup_fqn_func: Callable[[str], SymbolTableNode], tvar_scope: TypeVarScope, fail_func: Callable[[str, Context], None], + plugin: Plugin, allow_unnormalized: bool = False) -> Optional[Type]: """Return type if node is valid as a type alias rvalue. @@ -96,8 +98,8 @@ def analyze_type_alias(node: Expression, except TypeTranslationError: fail_func('Invalid type alias', node) return None - analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, tvar_scope, fail_func, aliasing=True, - allow_unnormalized=allow_unnormalized) + analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, tvar_scope, fail_func, plugin, + aliasing=True, allow_unnormalized=allow_unnormalized) return type.accept(analyzer) @@ -119,7 +121,8 @@ def __init__(self, lookup_func: Callable[[str, Context], SymbolTableNode], lookup_fqn_func: Callable[[str], SymbolTableNode], tvar_scope: TypeVarScope, - fail_func: Callable[[str, Context], None], *, + fail_func: Callable[[str, Context], None], + plugin: Plugin, *, aliasing: bool = False, allow_tuple_literal: bool = False, allow_unnormalized: bool = False) -> None: @@ -132,6 +135,7 @@ def __init__(self, # Positive if we are analyzing arguments of another (outer) type self.nesting_level = 0 self.allow_unnormalized = allow_unnormalized + self.plugin = plugin def visit_unbound_type(self, t: UnboundType) -> Type: if t.optional: @@ -147,6 +151,9 @@ def visit_unbound_type(self, t: UnboundType) -> Type: self.fail('Internal error (node is None, kind={})'.format(sym.kind), t) return AnyType() fullname = sym.node.fullname() + hook = self.plugin.get_type_analyze_hook(fullname) + if hook: + return hook(t, self.create_plugin_context(t)) if (fullname in nongen_builtins and t.args and not sym.normalized and not self.allow_unnormalized): self.fail(no_subscript_builtin_alias(fullname), t) @@ -400,37 +407,10 @@ def analyze_callable_type(self, t: UnboundType) -> Type: ret_type = t.args[1] if isinstance(t.args[0], TypeList): # Callable[[ARG, ...], RET] (ordinary callable type) - args = [] # type: List[Type] - names = [] # type: List[str] - kinds = [] # type: List[int] - for arg in t.args[0].items: - if isinstance(arg, CallableArgument): - args.append(arg.typ) - names.append(arg.name) - if arg.constructor is None: - return AnyType() - found = self.lookup(arg.constructor, arg) - if found is None: - # Looking it up already put an error message in - return AnyType() - elif found.fullname not in ARG_KINDS_BY_CONSTRUCTOR: - self.fail('Invalid argument constructor "{}"'.format( - found.fullname), arg) - return AnyType() - else: - kind = ARG_KINDS_BY_CONSTRUCTOR[found.fullname] - kinds.append(kind) - if arg.name is not None and kind in {ARG_STAR, ARG_STAR2}: - self.fail("{} arguments should not have names".format( - arg.constructor), arg) - return AnyType() - else: - args.append(arg) - names.append(None) - kinds.append(ARG_POS) - - check_arg_names(names, [t] * len(args), self.fail, "Callable") - check_arg_kinds(kinds, [t] * len(args), self.fail) + analyzed_args = self.analyze_callable_args(t.args[0]) + if analyzed_args is None: + return AnyType() + args, kinds, names = analyzed_args ret = CallableType(args, kinds, names, @@ -453,6 +433,44 @@ def analyze_callable_type(self, t: UnboundType) -> Type: assert isinstance(ret, CallableType) return ret.accept(self) + def analyze_callable_args(self, t: TypeList) -> Optional[Tuple[List[Type], + List[int], + List[Optional[str]]]]: + args = [] # type: List[Type] + kinds = [] # type: List[int] + names = [] # type: List[str] + for arg in t.items: + if isinstance(arg, CallableArgument): + args.append(arg.typ) + names.append(arg.name) + if arg.constructor is None: + return None + found = self.lookup(arg.constructor, arg) + if found is None: + # Looking it up already put an error message in + return None + elif found.fullname not in ARG_KINDS_BY_CONSTRUCTOR: + self.fail('Invalid argument constructor "{}"'.format( + found.fullname), arg) + return None + else: + kind = ARG_KINDS_BY_CONSTRUCTOR[found.fullname] + kinds.append(kind) + if arg.name is not None and kind in {ARG_STAR, ARG_STAR2}: + self.fail("{} arguments should not have names".format( + arg.constructor), arg) + return None + else: + args.append(arg) + kinds.append(ARG_POS) + names.append(None) + check_arg_names(names, [t] * len(args), self.fail, "Callable") + check_arg_kinds(kinds, [t] * len(args), self.fail) + return args, kinds, names + + def analyze_type(self, t: Type) -> Type: + return t.accept(self) + @contextmanager def tvar_scope_frame(self) -> Iterator[None]: old_scope = self.tvar_scope @@ -540,6 +558,10 @@ def builtin_type(self, fully_qualified_name: str, args: List[Type] = None) -> In def tuple_type(self, items: List[Type]) -> TupleType: return TupleType(items, fallback=self.builtin_type('builtins.tuple', [AnyType()])) + def create_plugin_context(self, context: Context) -> SemanticAnalysisPluginContext: + return SemanticAnalysisPluginContext( + self.builtin_type, self.fail, self.analyze_type, self.analyze_callable_args, context) + class TypeAnalyserPass3(TypeVisitor[None]): """Analyze type argument counts and values of generic types. diff --git a/mypy/types.py b/mypy/types.py index d8598554aeb4..1f45da1237ac 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -549,7 +549,7 @@ class CallableType(FunctionLike): is_var_arg = False # Is it a varargs function? derived from arg_kinds is_kw_arg = False ret_type = None # type: Type # Return value type - name = '' # type: Optional[str] # Name (may be None; for error messages) + name = '' # type: Optional[str] # Name (may be None; for error messages and plugins) definition = None # type: Optional[SymbolNode] # For error messages. May be None. # Type variables for a generic function variables = None # type: List[TypeVarDef] diff --git a/test-data/unit/check-custom-plugin.test b/test-data/unit/check-custom-plugin.test index 30b00a4b3a62..5799930092c4 100644 --- a/test-data/unit/check-custom-plugin.test +++ b/test-data/unit/check-custom-plugin.test @@ -69,3 +69,55 @@ f() plugins=/test-data/unit/plugins/badreturn2.py [out] /test-data/unit/plugins/badreturn2.py:0: error: Return value of "plugin" must be a subclass of "mypy.plugin.Plugin" + +[case testAttributeTypeHookPlugin] +# flags: --config-file tmp/mypy.ini +from typing import Callable +from m import Signal +s: Signal[Callable[[int], None]] = Signal() +s(1) +s('') # E: Argument 1 has incompatible type "str"; expected "int" +[file m.py] +from typing import TypeVar, Generic, Callable +T = TypeVar('T', bound=Callable[..., None]) +class Signal(Generic[T]): + __call__: Callable[..., None] +[file mypy.ini] +[[mypy] +plugins=/test-data/unit/plugins/attrhook.py + +[case testTypeAnalyzeHookPlugin] +# flags: --config-file tmp/mypy.ini +from typing import Callable +from mypy_extensions import DefaultArg +from m import Signal +s: Signal[[int, DefaultArg(str, 'x')]] = Signal() +reveal_type(s) # E: Revealed type is 'm.Signal[def (builtins.int, x: builtins.str =)]' +s.x # E: Signal[Callable[[int, str], None]] has no attribute "x" +ss: Signal[int, str] # E: Invalid "Signal" type (expected "Signal[[t, ...]]") +[file m.py] +from typing import TypeVar, Generic, Callable +T = TypeVar('T', bound=Callable[..., None]) +class Signal(Generic[T]): + __call__: Callable[..., None] +[file mypy.ini] +[[mypy] +plugins=/test-data/unit/plugins/type_anal_hook.py +[builtins fixtures/dict.pyi] + +[case testFunctionPluginHookForReturnedCallable] +# flags: --config-file tmp/mypy.ini +from m import decorator1, decorator2 +@decorator1() +def f() -> None: pass +@decorator2() +def g() -> None: pass +reveal_type(f) # E: Revealed type is 'def (*Any, **Any) -> builtins.str' +reveal_type(g) # E: Revealed type is 'def (*Any, **Any) -> builtins.int' +[file m.py] +from typing import Callable +def decorator1() -> Callable[..., Callable[..., int]]: pass +def decorator2() -> Callable[..., Callable[..., int]]: pass +[file mypy.ini] +[[mypy] +plugins=/test-data/unit/plugins/named_callable.py diff --git a/test-data/unit/plugins/attrhook.py b/test-data/unit/plugins/attrhook.py new file mode 100644 index 000000000000..54f96718b52f --- /dev/null +++ b/test-data/unit/plugins/attrhook.py @@ -0,0 +1,21 @@ +from typing import Optional + +from mypy.plugin import Plugin, AttributeHook +from mypy.types import Type, Instance + + +class AttrPlugin(Plugin): + def get_attribute_hook(self, fullname: str) -> Optional[AttributeHook]: + if fullname == 'm.Signal.__call__': + return signal_call_callback + return None + + +def signal_call_callback(object_type: Type, inferred_attribute: Type) -> Type: + if isinstance(object_type, Instance) and object_type.type.fullname() == 'm.Signal': + return object_type.args[0] + return inferred_attribute + + +def plugin(version): + return AttrPlugin diff --git a/test-data/unit/plugins/named_callable.py b/test-data/unit/plugins/named_callable.py new file mode 100644 index 000000000000..20802042dd66 --- /dev/null +++ b/test-data/unit/plugins/named_callable.py @@ -0,0 +1,28 @@ +from mypy.plugin import Plugin +from mypy.types import CallableType + + +class MyPlugin(Plugin): + def get_function_hook(self, fullname): + if fullname == 'm.decorator1': + return decorator_call_hook + if fullname == 'm._decorated': + return decorate_hook + return None + + +def decorator_call_hook(arg_types, args, inferred_return_type, named_generic_type): + if isinstance(inferred_return_type, CallableType): + return inferred_return_type.copy_modified(name='m._decorated') + return inferred_return_type + + +def decorate_hook(arg_types, args, inferred_return_type, named_generic_type): + if isinstance(inferred_return_type, CallableType): + return inferred_return_type.copy_modified( + ret_type=named_generic_type('builtins.str', [])) + return inferred_return_type + + +def plugin(version): + return MyPlugin diff --git a/test-data/unit/plugins/type_anal_hook.py b/test-data/unit/plugins/type_anal_hook.py new file mode 100644 index 000000000000..323e8a639e6d --- /dev/null +++ b/test-data/unit/plugins/type_anal_hook.py @@ -0,0 +1,38 @@ +from typing import Optional + +from mypy.plugin import Plugin, TypeAnalyzeHook, SemanticAnalysisPluginContext +from mypy.types import Type, UnboundType, TypeList, AnyType, NoneTyp, CallableType + + +class TypeAnalyzePlugin(Plugin): + def get_type_analyze_hook(self, fullname: str) -> Optional[TypeAnalyzeHook]: + if fullname == 'm.Signal': + return signal_type_analyze_callback + return None + + +def signal_type_analyze_callback( + typ: UnboundType, + context: SemanticAnalysisPluginContext) -> Type: + if (len(typ.args) != 1 + or not isinstance(typ.args[0], TypeList)): + context.fail('Invalid "Signal" type (expected "Signal[[t, ...]]")', context.context) + return AnyType() + + args = typ.args[0] + assert isinstance(args, TypeList) + analyzed = context.analyze_arg_list(args) + if analyzed is None: + return AnyType() # Error generated elsewhere + arg_types, arg_kinds, arg_names = analyzed + arg_types = [context.analyze_type(arg) for arg in arg_types] + type_arg = CallableType(arg_types, + arg_kinds, + arg_names, + NoneTyp(), + context.named_instance('builtins.function', [])) + return context.named_instance('m.Signal', [type_arg]) + + +def plugin(version): + return TypeAnalyzePlugin From 77bb797b719d6cb72ba498614c276f6166c0abcd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Jun 2017 13:19:53 +0100 Subject: [PATCH 02/10] Refactor based on feedback --- mypy/build.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 5e40af8ebe81..3d49f4f30760 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -173,7 +173,7 @@ def build(sources: List[BuildSource], reports = Reports(data_dir, options.report_dirs) source_set = BuildSourceSet(sources) errors = Errors(options.show_error_context, options.show_column_numbers) - plugin = load_custom_plugins(DefaultPlugin(options.python_version), options, errors) + plugin = load_plugins(options, errors) # Construct a build manager object to hold state during the build. # @@ -336,18 +336,18 @@ def import_priority(imp: ImportBase, toplevel_priority: int) -> int: return toplevel_priority -def load_custom_plugins(default_plugin: Plugin, options: Options, errors: Errors) -> Plugin: - """Load custom plugins if any are configured. +def load_plugins(options: Options, errors: Errors) -> Plugin: + """Load all configured plugins. - Return a plugin that chains all custom plugins (if any) and falls - back to default_plugin. + Return a plugin that encapsulates all plugins chained together. Always + at least include the default plugin. """ def plugin_error(message: str) -> None: errors.report(0, 0, message) errors.raise_error() - custom_plugins = [] + plugins = [DefaultPlugin(options.python_version)] # type: List[Plugin] for plugin_path in options.plugins: if options.config_file: # Plugin paths are relative to the config file location. @@ -386,15 +386,15 @@ def plugin_error(message: str) -> None: plugin_error( 'Return value of "plugin" must be a subclass of "mypy.plugin.Plugin"') try: - custom_plugins.append(plugin_type(options.python_version)) + plugins.append(plugin_type(options.python_version)) except Exception: print('Error constructing plugin instance of {}\n'.format(plugin_type.__name__)) raise # Propagate to display traceback - if not custom_plugins: - return default_plugin + if len(plugins) == 1: + return plugins[0] else: # Custom plugins take precendence over built-in plugins. - return ChainedPlugin(options.python_version, custom_plugins + [default_plugin]) + return ChainedPlugin(options.python_version, plugins) # TODO: Get rid of all_types. It's not used except for one log message. From 6471eaa5d22f67b06d511545fb39a38cc923bd92 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Jun 2017 13:21:49 +0100 Subject: [PATCH 03/10] Pass Options to plugins instead of just the Python version --- mypy/build.py | 6 +++--- mypy/plugin.py | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 3d49f4f30760..210d31077471 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -347,7 +347,7 @@ def plugin_error(message: str) -> None: errors.report(0, 0, message) errors.raise_error() - plugins = [DefaultPlugin(options.python_version)] # type: List[Plugin] + plugins = [DefaultPlugin(options)] # type: List[Plugin] for plugin_path in options.plugins: if options.config_file: # Plugin paths are relative to the config file location. @@ -386,7 +386,7 @@ def plugin_error(message: str) -> None: plugin_error( 'Return value of "plugin" must be a subclass of "mypy.plugin.Plugin"') try: - plugins.append(plugin_type(options.python_version)) + plugins.append(plugin_type(options)) except Exception: print('Error constructing plugin instance of {}\n'.format(plugin_type.__name__)) raise # Propagate to display traceback @@ -394,7 +394,7 @@ def plugin_error(message: str) -> None: return plugins[0] else: # Custom plugins take precendence over built-in plugins. - return ChainedPlugin(options.python_version, plugins) + return ChainedPlugin(options, plugins) # TODO: Get rid of all_types. It's not used except for one log message. diff --git a/mypy/plugin.py b/mypy/plugin.py index 20a0290c5422..f0aa450ea32f 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -6,6 +6,7 @@ AnyType, TypeList, UnboundType ) from mypy.messages import MessageBuilder +from mypy.options import Options # Create an Instance given full name of class and type arguments. @@ -105,8 +106,9 @@ class Plugin: results might be cached). """ - def __init__(self, python_version: Tuple[int, int]) -> None: - self.python_version = python_version + def __init__(self, options: Options) -> None: + self.options = options + self.python_version = options.python_version def get_function_hook(self, fullname: str) -> Optional[FunctionHook]: return None @@ -141,12 +143,12 @@ class ChainedPlugin(Plugin): # TODO: Support caching of lookup results (through a LRU cache, for example). - def __init__(self, python_version: Tuple[int, int], plugins: List[Plugin]) -> None: + def __init__(self, options: Options, plugins: List[Plugin]) -> None: """Initialize chained plugin. Assume that the child plugins aren't mutated (results may be cached). """ - super().__init__(python_version) + super().__init__(options) self._plugins = plugins def get_function_hook(self, fullname: str) -> Optional[FunctionHook]: From ae701416ec947e6a07f26df9164869b2c9613a72 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Jun 2017 13:45:22 +0100 Subject: [PATCH 04/10] Update based on feedback --- mypy/build.py | 2 +- mypy/checkmember.py | 16 ++++++---------- mypy/plugin.py | 35 ++++++++++++++--------------------- mypy/test/testgraph.py | 5 +++-- 4 files changed, 24 insertions(+), 34 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 210d31077471..b28dc330f98a 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -415,12 +415,12 @@ class BuildManager: semantic_analyzer_pass3: Semantic analyzer, pass 3 all_types: Map {Expression: Type} collected from all modules - errors: Used for reporting all errors options: Build options missing_modules: Set of modules that could not be imported encountered so far stale_modules: Set of modules that needed to be rechecked version_id: The current mypy version (based on commit id when possible) plugin: Active mypy plugin(s) + errors: Used for reporting all errors """ def __init__(self, data_dir: str, diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 8b4fec1e9f7f..3bc84d5428e0 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -37,8 +37,8 @@ def analyze_member_access(name: str, not_ready_callback: Callable[[str, Context], None], msg: MessageBuilder, *, original_type: Type, - override_info: TypeInfo = None, - chk: 'mypy.checker.TypeChecker' = None) -> Type: + chk: 'mypy.checker.TypeChecker', + override_info: TypeInfo = None) -> Type: """Return the type of attribute `name` of typ. This is a general operation that supports various different variations: @@ -77,9 +77,8 @@ def analyze_member_access(name: str, if method.is_property: assert isinstance(method, OverloadedFuncDef) first_item = cast(Decorator, method.items[0]) - plugin = chk.plugin if chk is not None else None return analyze_var(name, first_item.var, typ, info, node, is_lvalue, msg, - original_type, not_ready_callback, plugin) + original_type, not_ready_callback, chk.plugin) if is_lvalue: msg.cant_assign_to_method(node) signature = function_type(method, builtin_type('builtins.function')) @@ -104,7 +103,7 @@ def analyze_member_access(name: str, # The base object has dynamic type. return AnyType() elif isinstance(typ, NoneTyp): - if chk and chk.should_suppress_optional_error([typ]): + if chk.should_suppress_optional_error([typ]): return AnyType() # The only attribute NoneType has are those it inherits from object return analyze_member_access(name, builtin_type('builtins.object'), node, is_lvalue, @@ -202,7 +201,7 @@ def analyze_member_access(name: str, is_operator, builtin_type, not_ready_callback, msg, original_type=original_type, chk=chk) - if chk and chk.should_suppress_optional_error([typ]): + if chk.should_suppress_optional_error([typ]): return AnyType() return msg.has_no_attr(original_type, typ, name, node) @@ -293,6 +292,7 @@ def analyze_var(name: str, var: Var, itype: Instance, info: TypeInfo, node: Cont msg.read_only_property(name, info, node) if is_lvalue and var.is_classvar: msg.cant_assign_to_classvar(name, node) + result = t if var.is_initialized_in_class and isinstance(t, FunctionLike) and not t.is_type_obj(): if is_lvalue: if var.is_property: @@ -315,10 +315,6 @@ def analyze_var(name: str, var: Var, itype: Instance, info: TypeInfo, node: Cont result = signature.ret_type else: result = signature - else: - result = t - else: - result = t else: if not var.is_ready: not_ready_callback(var.name(), node) diff --git a/mypy/plugin.py b/mypy/plugin.py index f0aa450ea32f..2f5d93302240 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -19,24 +19,17 @@ # Some objects and callbacks that plugins can use to get information from the # type checker or to report errors. PluginContext = NamedTuple( - 'PluginContext', - [ - ('named_instance', NamedInstanceCallback), - ('msg', MessageBuilder), - ('context', Context) - ] -) + 'PluginContext', [('named_instance', NamedInstanceCallback), + ('msg', MessageBuilder), + ('context', Context)]) +# Like above, for plugins that run during semantic analysis. SemanticAnalysisPluginContext = NamedTuple( - 'SemanticAnalysisPluginContext', - [ - ('named_instance', NamedInstanceCallback), - ('fail', Callable[[str, Context], None]), - ('analyze_type', Callable[[Type], Type]), - ('analyze_arg_list', AnalyzeArgListCallback), - ('context', Context) - ] -) + 'SemanticAnalysisPluginContext', [('named_instance', NamedInstanceCallback), + ('fail', Callable[[str, Context], None]), + ('analyze_type', Callable[[Type], Type]), + ('analyze_arg_list', AnalyzeArgListCallback), + ('context', Context)]) # A callback that infers the return type of a function with a special signature. # @@ -47,7 +40,7 @@ List[List[Type]], # List of types caller provides for each formal argument List[List[Expression]], # Actual argument expressions for each formal argument Type, # Return type for call inferred using the regular signature - NamedInstanceCallback # Callable for constructing a named instance type + NamedInstanceCallback, # Callable for constructing a named instance type ], Type # Return type inferred by the callback ] @@ -59,7 +52,7 @@ Type, # Base object type List[List[Expression]], # Actual argument expressions for each formal argument CallableType, # Original signature of the method - NamedInstanceCallback # Callable for constructing a named instance type + NamedInstanceCallback, # Callable for constructing a named instance type ], CallableType # Potentially more precise signature inferred for the method ] @@ -73,7 +66,7 @@ List[List[Type]], # List of types caller provides for each formal argument List[List[Expression]], # Actual argument expressions for each formal argument Type, # Return type for call inferred using the regular signature - PluginContext # Access to type checking context + PluginContext, # Access to type checking context ], Type # Return type inferred by the callback ] @@ -81,7 +74,7 @@ AttributeHook = Callable[ [ Type, # Base object type - Type # Inferred attribute type + Type, # Inferred attribute type # TODO: Some context object? ], Type @@ -90,7 +83,7 @@ TypeAnalyzeHook = Callable[ [ UnboundType, - SemanticAnalysisPluginContext + SemanticAnalysisPluginContext, ], Type ] diff --git a/mypy/test/testgraph.py b/mypy/test/testgraph.py index 58c4e8b3a9f6..dbbe4872aa75 100644 --- a/mypy/test/testgraph.py +++ b/mypy/test/testgraph.py @@ -38,15 +38,16 @@ def test_scc(self) -> None: def _make_manager(self) -> BuildManager: errors = Errors() + options = Options() manager = BuildManager( data_dir='', lib_path=[], ignore_prefix='', source_set=BuildSourceSet([]), reports=Reports('', {}), - options=Options(), + options=options, version_id=__version__, - plugin=Plugin(defaults.PYTHON3_VERSION), + plugin=Plugin(options), errors=errors, ) return manager From 3283cbf6da5404e8b0041b6d5077fc5f09e28d63 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Jun 2017 16:09:27 +0100 Subject: [PATCH 05/10] Major redesign of the plugin system Instead of passing several arguments to hook function, always pass just a single object. This simplifies the signatures of hooks. Instead of passing callback functions to hooks, pass an object that implements a specific interface. These changes are intended to make it easier to write plugins, and to make it easier to evolve the plugin system. Adding extra attributes to context or extra methods to the internal interfaces doesn't require changes to existing plugins. --- mypy/checker.py | 4 +- mypy/checkexpr.py | 27 +- mypy/checkmember.py | 20 +- mypy/plugin.py | 305 ++++++++++++----------- mypy/typeanal.py | 31 ++- test-data/unit/plugins/attrhook.py | 14 +- test-data/unit/plugins/fnplugin.py | 4 +- test-data/unit/plugins/named_callable.py | 18 +- test-data/unit/plugins/plugin2.py | 4 +- test-data/unit/plugins/type_anal_hook.py | 27 +- 10 files changed, 228 insertions(+), 226 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 98bb35474bf9..ed6d0dc0196d 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -57,7 +57,7 @@ from mypy.binder import ConditionalTypeBinder, get_declaration from mypy.meet import is_overlapping_types from mypy.options import Options -from mypy.plugin import Plugin +from mypy.plugin import Plugin, CheckerPluginInterface from mypy import experiments @@ -80,7 +80,7 @@ ]) -class TypeChecker(NodeVisitor[None]): +class TypeChecker(NodeVisitor[None], CheckerPluginInterface): """Mypy type checker. Type check mypy source files that have been semantically analyzed. diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index e291a6eb3a9a..0b7975b7238a 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -45,7 +45,7 @@ from mypy.util import split_module_names from mypy.typevars import fill_typevars from mypy.visitor import ExpressionVisitor -from mypy.plugin import Plugin, PluginContext, MethodSignatureHook +from mypy.plugin import Plugin, MethodContext, MethodSigContext, FunctionContext from mypy.typeanal import make_optional_type from mypy import experiments @@ -392,17 +392,21 @@ def apply_function_plugin(self, # Apply function plugin callback = self.plugin.get_function_hook(fullname) assert callback is not None # Assume that caller ensures this - return callback(formal_arg_types, formal_arg_exprs, inferred_ret_type, - self.chk.named_generic_type) + return callback( + FunctionContext(formal_arg_types, inferred_ret_type, formal_arg_exprs, + context, self.chk)) else: # Apply method plugin method_callback = self.plugin.get_method_hook(fullname) assert method_callback is not None # Assume that caller ensures this - return method_callback(object_type, formal_arg_types, formal_arg_exprs, - inferred_ret_type, self.create_plugin_context(context)) - - def apply_method_signature_hook(self, e: CallExpr, callee: FunctionLike, object_type: Type, - signature_hook: MethodSignatureHook) -> FunctionLike: + return method_callback( + MethodContext(object_type, formal_arg_types, + inferred_ret_type, formal_arg_exprs, + context, self.chk)) + + def apply_method_signature_hook( + self, e: CallExpr, callee: FunctionLike, object_type: Type, + signature_hook: Callable[[MethodSigContext], CallableType]) -> FunctionLike: """Apply a plugin hook that may infer a more precise signature for a method.""" if isinstance(callee, CallableType): arg_kinds = e.arg_kinds @@ -417,8 +421,8 @@ def apply_method_signature_hook(self, e: CallExpr, callee: FunctionLike, object_ for formal, actuals in enumerate(formal_to_actual): for actual in actuals: formal_arg_exprs[formal].append(args[actual]) - return signature_hook(object_type, formal_arg_exprs, callee, - self.chk.named_generic_type) + return signature_hook( + MethodSigContext(object_type, formal_arg_exprs, callee, e, self.chk)) else: assert isinstance(callee, Overloaded) items = [] @@ -428,9 +432,6 @@ def apply_method_signature_hook(self, e: CallExpr, callee: FunctionLike, object_ items.append(adjusted) return Overloaded(items) - def create_plugin_context(self, context: Context) -> PluginContext: - return PluginContext(self.chk.named_generic_type, self.msg, context) - def check_call_expr_with_callee_type(self, callee_type: Type, e: CallExpr, diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 3bc84d5428e0..d3f58a3a5917 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -17,7 +17,7 @@ from mypy.expandtype import expand_type_by_instance, expand_type, freshen_function_type_vars from mypy.infer import infer_type_arguments from mypy.typevars import fill_typevars -from mypy.plugin import Plugin +from mypy.plugin import Plugin, AttributeContext from mypy import messages from mypy import subtypes MYPY = False @@ -78,7 +78,7 @@ def analyze_member_access(name: str, assert isinstance(method, OverloadedFuncDef) first_item = cast(Decorator, method.items[0]) return analyze_var(name, first_item.var, typ, info, node, is_lvalue, msg, - original_type, not_ready_callback, chk.plugin) + original_type, not_ready_callback, chk=chk) if is_lvalue: msg.cant_assign_to_method(node) signature = function_type(method, builtin_type('builtins.function')) @@ -228,9 +228,8 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo, v = vv.var if isinstance(v, Var): - plugin = chk.plugin if chk is not None else None return analyze_var(name, v, itype, info, node, is_lvalue, msg, - original_type, not_ready_callback, plugin) + original_type, not_ready_callback, chk=chk) elif isinstance(v, FuncDef): assert False, "Did not expect a function" elif not v and name not in ['__getattr__', '__setattr__', '__getattribute__']: @@ -272,8 +271,8 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo, def analyze_var(name: str, var: Var, itype: Instance, info: TypeInfo, node: Context, is_lvalue: bool, msg: MessageBuilder, original_type: Type, - not_ready_callback: Callable[[str, Context], None], - plugin: Optional[Plugin]) -> Type: + not_ready_callback: Callable[[str, Context], None], *, + chk: 'mypy.checker.TypeChecker') -> Type: """Analyze access to an attribute via a Var node. This is conceptually part of analyze_member_access and the arguments are similar. @@ -320,11 +319,10 @@ def analyze_var(name: str, var: Var, itype: Instance, info: TypeInfo, node: Cont not_ready_callback(var.name(), node) # Implicit 'Any' type. result = AnyType() - if plugin: - fullname = '{}.{}'.format(var.info.fullname(), name) - hook = plugin.get_attribute_hook(fullname) - if hook: - result = hook(original_type, result) + fullname = '{}.{}'.format(var.info.fullname(), name) + hook = chk.plugin.get_attribute_hook(fullname) + if hook: + result = hook(AttributeContext(original_type, result, node, chk)) return result diff --git a/mypy/plugin.py b/mypy/plugin.py index 2f5d93302240..848d12233551 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -1,3 +1,6 @@ +"""Plugin system for extending mypy.""" + +from abc import abstractmethod from typing import Callable, List, Tuple, Optional, NamedTuple, TypeVar from mypy.nodes import Expression, StrExpr, IntExpr, UnaryExpr, Context @@ -9,84 +12,90 @@ from mypy.options import Options -# Create an Instance given full name of class and type arguments. -NamedInstanceCallback = Callable[[str, List[Type]], Type] +class AnalyzerPluginInterface: + """Interface for accessing semantic analyzer functionality from plugins.""" + + @abstractmethod + def fail(self, msg: str, ctx: Context) -> None: + raise NotImplementedError + + @abstractmethod + def named_type(self, name: str, args: List[Type]) -> Instance: + raise NotImplementedError + + @abstractmethod + def analyze_type(self, typ: Type) -> Type: + raise NotImplementedError + + @abstractmethod + def analyze_callable_args(self, args: TypeList) -> Optional[Tuple[List[Type], + List[int], + List[Optional[str]]]]: + raise NotImplementedError + + +# A context for a hook that semantically analyzes an unbound type. +AnalyzeTypeContext = NamedTuple( + 'AnalyzeTypeContext', [ + ('type', UnboundType), # Type to analyze + ('context', Context), + ('api', AnalyzerPluginInterface)]) -AnalyzeArgListCallback = Callable[[TypeList], Optional[Tuple[List[Type], - List[int], - List[Optional[str]]]]] -# Some objects and callbacks that plugins can use to get information from the -# type checker or to report errors. -PluginContext = NamedTuple( - 'PluginContext', [('named_instance', NamedInstanceCallback), - ('msg', MessageBuilder), - ('context', Context)]) +class CheckerPluginInterface: + """Interface for accessing type checker functionality from plugins.""" -# Like above, for plugins that run during semantic analysis. -SemanticAnalysisPluginContext = NamedTuple( - 'SemanticAnalysisPluginContext', [('named_instance', NamedInstanceCallback), - ('fail', Callable[[str, Context], None]), - ('analyze_type', Callable[[Type], Type]), - ('analyze_arg_list', AnalyzeArgListCallback), - ('context', Context)]) + msg = None # type: MessageBuilder -# A callback that infers the return type of a function with a special signature. + @abstractmethod + def named_generic_type(self, name: str, args: List[Type]) -> Instance: + raise NotImplementedError + + +# A context for a function hook that infers the return type of a function with +# a special signature. # -# A no-op callback would just return the inferred return type, but a useful callback -# at least sometimes can infer a more precise type. -FunctionHook = Callable[ - [ - List[List[Type]], # List of types caller provides for each formal argument - List[List[Expression]], # Actual argument expressions for each formal argument - Type, # Return type for call inferred using the regular signature - NamedInstanceCallback, # Callable for constructing a named instance type - ], - Type # Return type inferred by the callback -] - -# A callback that may infer a better signature for a method. Note that argument types aren't -# available yet. If you need them, you have to use a MethodHook instead. -MethodSignatureHook = Callable[ - [ - Type, # Base object type - List[List[Expression]], # Actual argument expressions for each formal argument - CallableType, # Original signature of the method - NamedInstanceCallback, # Callable for constructing a named instance type - ], - CallableType # Potentially more precise signature inferred for the method -] - -# A callback that infers the return type of a method with a special signature. +# A no-op callback would just return the inferred return type, but a useful +# callback at least sometimes can infer a more precise type. +FunctionContext = NamedTuple( + 'FunctionContext', [ + ('arg_types', List[List[Type]]), # List of actual caller types for each formal argument + ('inferred_return_type', Type), # Return type inferred from signature + ('args', List[List[Expression]]), # Actual expressions for each formal argument + ('context', Context), + ('api', CheckerPluginInterface)]) + +# A context for a method signature hook that infers a better signature for a +# method. Note that argument types aren't available yet. If you need them, +# you have to use a method hook instead. +MethodSigContext = NamedTuple( + 'MethodSigContext', [ + ('type', Type), # Base object type for method call + ('args', List[List[Expression]]), # Actual expressions for each formal argument + ('signature', CallableType), # Original signature of the method + ('context', Context), + ('api', CheckerPluginInterface)]) + +# A context for a method hook that infers the return type of a method with a +# special signature. # -# This is pretty similar to FunctionHook. -MethodHook = Callable[ - [ - Type, # Base object type - List[List[Type]], # List of types caller provides for each formal argument - List[List[Expression]], # Actual argument expressions for each formal argument - Type, # Return type for call inferred using the regular signature - PluginContext, # Access to type checking context - ], - Type # Return type inferred by the callback -] - -AttributeHook = Callable[ - [ - Type, # Base object type - Type, # Inferred attribute type - # TODO: Some context object? - ], - Type -] - -TypeAnalyzeHook = Callable[ - [ - UnboundType, - SemanticAnalysisPluginContext, - ], - Type -] +# This is very similar to FunctionContext (only differences are documented). +MethodContext = NamedTuple( + 'MethodContext', [ + ('type', Type), # Base object type for method call + ('arg_types', List[List[Type]]), + ('inferred_return_type', Type), + ('args', List[List[Expression]]), + ('context', Context), + ('api', CheckerPluginInterface)]) + +# A context for an attribute type hook that infers the type of an attribute. +AttributeContext = NamedTuple( + 'AttributeContext', [ + ('type', Type), # Type of object with attribute + ('inferred_attr_type', Type), # Original attribute type + ('context', Context), + ('api', CheckerPluginInterface)]) class Plugin: @@ -97,25 +106,33 @@ class Plugin: All get_ methods are treated as pure functions (you should assume that results might be cached). + + Look at the comments of various *Context objects for descriptions of + various hooks. """ def __init__(self, options: Options) -> None: self.options = options self.python_version = options.python_version - def get_function_hook(self, fullname: str) -> Optional[FunctionHook]: + def get_type_analyze_hook(self, fullname: str + ) -> Optional[Callable[[AnalyzeTypeContext], Type]]: return None - def get_method_signature_hook(self, fullname: str) -> Optional[MethodSignatureHook]: + def get_function_hook(self, fullname: str + ) -> Optional[Callable[[FunctionContext], Type]]: return None - def get_method_hook(self, fullname: str) -> Optional[MethodHook]: + def get_method_signature_hook(self, fullname: str + ) -> Optional[Callable[[MethodSigContext], CallableType]]: return None - def get_attribute_hook(self, fullname: str) -> Optional[AttributeHook]: + def get_method_hook(self, fullname: str + ) -> Optional[Callable[[MethodContext], Type]]: return None - def get_type_analyze_hook(self, fullname: str) -> Optional[TypeAnalyzeHook]: + def get_attribute_hook(self, fullname: str + ) -> Optional[Callable[[AttributeContext], Type]]: return None # TODO: metaclass / class decorator hook @@ -144,21 +161,26 @@ def __init__(self, options: Options, plugins: List[Plugin]) -> None: super().__init__(options) self._plugins = plugins - def get_function_hook(self, fullname: str) -> Optional[FunctionHook]: + def get_type_analyze_hook(self, fullname: str + ) -> Optional[Callable[[AnalyzeTypeContext], Type]]: + return self._find_hook(lambda plugin: plugin.get_type_analyze_hook(fullname)) + + def get_function_hook(self, fullname: str + ) -> Optional[Callable[[FunctionContext], Type]]: return self._find_hook(lambda plugin: plugin.get_function_hook(fullname)) - def get_method_signature_hook(self, fullname: str) -> Optional[MethodSignatureHook]: + def get_method_signature_hook(self, fullname: str + ) -> Optional[Callable[[MethodSigContext], CallableType]]: return self._find_hook(lambda plugin: plugin.get_method_signature_hook(fullname)) - def get_method_hook(self, fullname: str) -> Optional[MethodHook]: + def get_method_hook(self, fullname: str + ) -> Optional[Callable[[MethodContext], Type]]: return self._find_hook(lambda plugin: plugin.get_method_hook(fullname)) - def get_attribute_hook(self, fullname: str) -> Optional[AttributeHook]: + def get_attribute_hook(self, fullname: str + ) -> Optional[Callable[[AttributeContext], Type]]: return self._find_hook(lambda plugin: plugin.get_attribute_hook(fullname)) - def get_type_analyze_hook(self, fullname: str) -> Optional[TypeAnalyzeHook]: - return self._find_hook(lambda plugin: plugin.get_type_analyze_hook(fullname)) - def _find_hook(self, lookup: Callable[[Plugin], T]) -> Optional[T]: for plugin in self._plugins: hook = lookup(plugin) @@ -170,19 +192,22 @@ def _find_hook(self, lookup: Callable[[Plugin], T]) -> Optional[T]: class DefaultPlugin(Plugin): """Type checker plugin that is enabled by default.""" - def get_function_hook(self, fullname: str) -> Optional[FunctionHook]: + def get_function_hook(self, fullname: str + ) -> Optional[Callable[[FunctionContext], Type]]: if fullname == 'contextlib.contextmanager': return contextmanager_callback elif fullname == 'builtins.open' and self.python_version[0] == 3: return open_callback return None - def get_method_signature_hook(self, fullname: str) -> Optional[MethodSignatureHook]: + def get_method_signature_hook(self, fullname: str + ) -> Optional[Callable[[MethodSigContext], CallableType]]: if fullname == 'typing.Mapping.get': return typed_dict_get_signature_callback return None - def get_method_hook(self, fullname: str) -> Optional[MethodHook]: + def get_method_hook(self, fullname: str + ) -> Optional[Callable[[MethodContext], Type]]: if fullname == 'typing.Mapping.get': return typed_dict_get_callback elif fullname == 'builtins.int.__pow__': @@ -190,67 +215,57 @@ def get_method_hook(self, fullname: str) -> Optional[MethodHook]: return None -def open_callback( - arg_types: List[List[Type]], - args: List[List[Expression]], - inferred_return_type: Type, - named_generic_type: Callable[[str, List[Type]], Type]) -> Type: +def open_callback(ctx: FunctionContext) -> Type: """Infer a better return type for 'open'. Infer TextIO or BinaryIO as the return value if the mode argument is not given or is a literal. """ mode = None - if not arg_types or len(arg_types[1]) != 1: + if not ctx.arg_types or len(ctx.arg_types[1]) != 1: mode = 'r' - elif isinstance(args[1][0], StrExpr): - mode = args[1][0].value + elif isinstance(ctx.args[1][0], StrExpr): + mode = ctx.args[1][0].value if mode is not None: - assert isinstance(inferred_return_type, Instance) + assert isinstance(ctx.inferred_return_type, Instance) if 'b' in mode: - return named_generic_type('typing.BinaryIO', []) + return ctx.api.named_generic_type('typing.BinaryIO', []) else: - return named_generic_type('typing.TextIO', []) - return inferred_return_type + return ctx.api.named_generic_type('typing.TextIO', []) + return ctx.inferred_return_type -def contextmanager_callback( - arg_types: List[List[Type]], - args: List[List[Expression]], - inferred_return_type: Type, - named_generic_type: Callable[[str, List[Type]], Type]) -> Type: +def contextmanager_callback(ctx: FunctionContext) -> Type: """Infer a better return type for 'contextlib.contextmanager'.""" # Be defensive, just in case. - if arg_types and len(arg_types[0]) == 1: - arg_type = arg_types[0][0] - if isinstance(arg_type, CallableType) and isinstance(inferred_return_type, CallableType): + if ctx.arg_types and len(ctx.arg_types[0]) == 1: + arg_type = ctx.arg_types[0][0] + if (isinstance(arg_type, CallableType) + and isinstance(ctx.inferred_return_type, CallableType)): # The stub signature doesn't preserve information about arguments so # add them back here. - return inferred_return_type.copy_modified( + return ctx.inferred_return_type.copy_modified( arg_types=arg_type.arg_types, arg_kinds=arg_type.arg_kinds, arg_names=arg_type.arg_names) - return inferred_return_type + return ctx.inferred_return_type -def typed_dict_get_signature_callback( - object_type: Type, - args: List[List[Expression]], - signature: CallableType, - named_generic_type: Callable[[str, List[Type]], Type]) -> CallableType: +def typed_dict_get_signature_callback(ctx: MethodSigContext) -> CallableType: """Try to infer a better signature type for TypedDict.get. This is used to get better type context for the second argument that depends on a TypedDict value type. """ - if (isinstance(object_type, TypedDictType) - and len(args) == 2 - and len(args[0]) == 1 - and isinstance(args[0][0], StrExpr) + signature = ctx.signature + if (isinstance(ctx.type, TypedDictType) + and len(ctx.args) == 2 + and len(ctx.args[0]) == 1 + and isinstance(ctx.args[0][0], StrExpr) and len(signature.arg_types) == 2 and len(signature.variables) == 1): - key = args[0][0].value - value_type = object_type.items.get(key) + key = ctx.args[0][0].value + value_type = ctx.type.items.get(key) if value_type: # Tweak the signature to include the value type as context. It's # only needed for type inference since there's a union with a type @@ -262,49 +277,39 @@ def typed_dict_get_signature_callback( return signature -def typed_dict_get_callback( - object_type: Type, - arg_types: List[List[Type]], - args: List[List[Expression]], - inferred_return_type: Type, - context: PluginContext) -> Type: +def typed_dict_get_callback(ctx: MethodContext) -> Type: """Infer a precise return type for TypedDict.get with literal first argument.""" - if (isinstance(object_type, TypedDictType) - and len(arg_types) >= 1 - and len(arg_types[0]) == 1): - if isinstance(args[0][0], StrExpr): - key = args[0][0].value - value_type = object_type.items.get(key) + if (isinstance(ctx.type, TypedDictType) + and len(ctx.arg_types) >= 1 + and len(ctx.arg_types[0]) == 1): + if isinstance(ctx.args[0][0], StrExpr): + key = ctx.args[0][0].value + value_type = ctx.type.items.get(key) if value_type: - if len(arg_types) == 1: + if len(ctx.arg_types) == 1: return UnionType.make_simplified_union([value_type, NoneTyp()]) - elif len(arg_types) == 2 and len(arg_types[1]) == 1: - return UnionType.make_simplified_union([value_type, arg_types[1][0]]) + elif len(ctx.arg_types) == 2 and len(ctx.arg_types[1]) == 1: + return UnionType.make_simplified_union([value_type, ctx.arg_types[1][0]]) else: - context.msg.typeddict_item_name_not_found(object_type, key, context.context) + ctx.api.msg.typeddict_item_name_not_found(ctx.type, key, ctx.context) return AnyType() - return inferred_return_type + return ctx.inferred_return_type -def int_pow_callback( - object_type: Type, - arg_types: List[List[Type]], - args: List[List[Expression]], - inferred_return_type: Type, - context: PluginContext) -> Type: +def int_pow_callback(ctx: MethodContext) -> Type: """Infer a more precise return type for int.__pow__.""" - if (len(arg_types) == 1 - and len(arg_types[0]) == 1): - arg = args[0][0] + if (len(ctx.arg_types) == 1 + and len(ctx.arg_types[0]) == 1): + arg = ctx.args[0][0] if isinstance(arg, IntExpr): exponent = arg.value elif isinstance(arg, UnaryExpr) and arg.op == '-' and isinstance(arg.expr, IntExpr): exponent = -arg.expr.value else: # Right operand not an int literal or a negated literal -- give up. - return inferred_return_type + return ctx.inferred_return_type if exponent >= 0: - return context.named_instance('builtins.int', []) + return ctx.api.named_generic_type('builtins.int', []) else: - return context.named_instance('builtins.float', []) - return inferred_return_type + return ctx.api.named_generic_type('builtins.float', []) + return ctx.inferred_return_type diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 9ec50a764fe4..2b37e531e340 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -24,7 +24,7 @@ from mypy.sametypes import is_same_type from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError from mypy.subtypes import is_subtype -from mypy.plugin import Plugin, SemanticAnalysisPluginContext +from mypy.plugin import Plugin, AnalyzerPluginInterface, AnalyzeTypeContext from mypy import nodes from mypy import experiments @@ -111,7 +111,7 @@ def no_subscript_builtin_alias(name: str, propose_alt: bool = True) -> str: return msg -class TypeAnalyser(SyntheticTypeVisitor[Type]): +class TypeAnalyser(SyntheticTypeVisitor[Type], AnalyzerPluginInterface): """Semantic analyzer for types (semantic analysis pass 2). Converts unbound types into bound types. @@ -128,7 +128,7 @@ def __init__(self, allow_unnormalized: bool = False) -> None: self.lookup = lookup_func self.lookup_fqn_func = lookup_fqn_func - self.fail = fail_func + self.fail_func = fail_func self.tvar_scope = tvar_scope self.aliasing = aliasing self.allow_tuple_literal = allow_tuple_literal @@ -153,7 +153,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: fullname = sym.node.fullname() hook = self.plugin.get_type_analyze_hook(fullname) if hook: - return hook(t, self.create_plugin_context(t)) + return hook(AnalyzeTypeContext(t, t, self)) if (fullname in nongen_builtins and t.args and not sym.normalized and not self.allow_unnormalized): self.fail(no_subscript_builtin_alias(fullname), t) @@ -170,10 +170,10 @@ def visit_unbound_type(self, t: UnboundType) -> Type: elif fullname == 'typing.Tuple': if len(t.args) == 0 and not t.empty_tuple_index: # Bare 'Tuple' is same as 'tuple' - return self.builtin_type('builtins.tuple') + return self.named_type('builtins.tuple') if len(t.args) == 2 and isinstance(t.args[1], EllipsisType): # Tuple[T, ...] (uniform, variable-length tuple) - instance = self.builtin_type('builtins.tuple', [self.anal_type(t.args[0])]) + instance = self.named_type('builtins.tuple', [self.anal_type(t.args[0])]) instance.line = t.line return instance return self.tuple_type(self.anal_array(t.args)) @@ -348,7 +348,7 @@ def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: variables = self.bind_function_type_variables(t, t) ret = t.copy_modified(arg_types=self.anal_array(t.arg_types, nested=nested), ret_type=self.anal_type(t.ret_type, nested=nested), - fallback=t.fallback or self.builtin_type('builtins.function'), + fallback=t.fallback or self.named_type('builtins.function'), variables=self.anal_var_defs(variables)) return ret @@ -363,11 +363,11 @@ def visit_tuple_type(self, t: TupleType) -> Type: self.fail('At most one star type allowed in a tuple', t) if t.implicit: return TupleType([AnyType() for _ in t.items], - self.builtin_type('builtins.tuple'), + self.named_type('builtins.tuple'), t.line) else: return AnyType() - fallback = t.fallback if t.fallback else self.builtin_type('builtins.tuple', [AnyType()]) + fallback = t.fallback if t.fallback else self.named_type('builtins.tuple', [AnyType()]) return TupleType(self.anal_array(t.items), fallback, t.line) def visit_typeddict_type(self, t: TypedDictType) -> Type: @@ -394,7 +394,7 @@ def visit_type_type(self, t: TypeType) -> Type: return TypeType.make_normalized(self.anal_type(t.item), line=t.line) def analyze_callable_type(self, t: UnboundType) -> Type: - fallback = self.builtin_type('builtins.function') + fallback = self.named_type('builtins.function') if len(t.args) == 0: # Callable (bare). Treat as Callable[..., Any]. ret = CallableType([AnyType(), AnyType()], @@ -471,6 +471,9 @@ def analyze_callable_args(self, t: TypeList) -> Optional[Tuple[List[Type], def analyze_type(self, t: Type) -> Type: return t.accept(self) + def fail(self, msg: str, ctx: Context) -> None: + self.fail_func(msg, ctx) + @contextmanager def tvar_scope_frame(self) -> Iterator[None]: old_scope = self.tvar_scope @@ -550,17 +553,13 @@ def anal_var_defs(self, var_defs: List[TypeVarDef]) -> List[TypeVarDef]: vd.line)) return a - def builtin_type(self, fully_qualified_name: str, args: List[Type] = None) -> Instance: + def named_type(self, fully_qualified_name: str, args: List[Type] = None) -> Instance: node = self.lookup_fqn_func(fully_qualified_name) assert isinstance(node.node, TypeInfo) return Instance(node.node, args or []) def tuple_type(self, items: List[Type]) -> TupleType: - return TupleType(items, fallback=self.builtin_type('builtins.tuple', [AnyType()])) - - def create_plugin_context(self, context: Context) -> SemanticAnalysisPluginContext: - return SemanticAnalysisPluginContext( - self.builtin_type, self.fail, self.analyze_type, self.analyze_callable_args, context) + return TupleType(items, fallback=self.named_type('builtins.tuple', [AnyType()])) class TypeAnalyserPass3(TypeVisitor[None]): diff --git a/test-data/unit/plugins/attrhook.py b/test-data/unit/plugins/attrhook.py index 54f96718b52f..d94a5d6ccf79 100644 --- a/test-data/unit/plugins/attrhook.py +++ b/test-data/unit/plugins/attrhook.py @@ -1,20 +1,20 @@ -from typing import Optional +from typing import Optional, Callable -from mypy.plugin import Plugin, AttributeHook +from mypy.plugin import Plugin, AttributeContext from mypy.types import Type, Instance class AttrPlugin(Plugin): - def get_attribute_hook(self, fullname: str) -> Optional[AttributeHook]: + def get_attribute_hook(self, fullname: str) -> Optional[Callable[[AttributeContext], Type]]: if fullname == 'm.Signal.__call__': return signal_call_callback return None -def signal_call_callback(object_type: Type, inferred_attribute: Type) -> Type: - if isinstance(object_type, Instance) and object_type.type.fullname() == 'm.Signal': - return object_type.args[0] - return inferred_attribute +def signal_call_callback(ctx: AttributeContext) -> Type: + if isinstance(ctx.type, Instance) and ctx.type.type.fullname() == 'm.Signal': + return ctx.type.args[0] + return ctx.inferred_attr_type def plugin(version): diff --git a/test-data/unit/plugins/fnplugin.py b/test-data/unit/plugins/fnplugin.py index d5027219a09f..513279213b7d 100644 --- a/test-data/unit/plugins/fnplugin.py +++ b/test-data/unit/plugins/fnplugin.py @@ -6,8 +6,8 @@ def get_function_hook(self, fullname): return my_hook return None -def my_hook(arg_types, args, inferred_return_type, named_generic_type): - return named_generic_type('builtins.int', []) +def my_hook(ctx): + return ctx.api.named_generic_type('builtins.int', []) def plugin(version): return MyPlugin diff --git a/test-data/unit/plugins/named_callable.py b/test-data/unit/plugins/named_callable.py index 20802042dd66..51ed7239262f 100644 --- a/test-data/unit/plugins/named_callable.py +++ b/test-data/unit/plugins/named_callable.py @@ -11,17 +11,17 @@ def get_function_hook(self, fullname): return None -def decorator_call_hook(arg_types, args, inferred_return_type, named_generic_type): - if isinstance(inferred_return_type, CallableType): - return inferred_return_type.copy_modified(name='m._decorated') - return inferred_return_type +def decorator_call_hook(ctx): + if isinstance(ctx.inferred_return_type, CallableType): + return ctx.inferred_return_type.copy_modified(name='m._decorated') + return ctx.inferred_return_type -def decorate_hook(arg_types, args, inferred_return_type, named_generic_type): - if isinstance(inferred_return_type, CallableType): - return inferred_return_type.copy_modified( - ret_type=named_generic_type('builtins.str', [])) - return inferred_return_type +def decorate_hook(ctx): + if isinstance(ctx.inferred_return_type, CallableType): + return ctx.inferred_return_type.copy_modified( + ret_type=ctx.api.named_generic_type('builtins.str', [])) + return ctx.inferred_return_type def plugin(version): diff --git a/test-data/unit/plugins/plugin2.py b/test-data/unit/plugins/plugin2.py index 1584871fae1d..b530a62d23aa 100644 --- a/test-data/unit/plugins/plugin2.py +++ b/test-data/unit/plugins/plugin2.py @@ -6,8 +6,8 @@ def get_function_hook(self, fullname): return str_hook return None -def str_hook(arg_types, args, inferred_return_type, named_generic_type): - return named_generic_type('builtins.str', []) +def str_hook(ctx): + return ctx.api.named_generic_type('builtins.str', []) def plugin(version): return Plugin2 diff --git a/test-data/unit/plugins/type_anal_hook.py b/test-data/unit/plugins/type_anal_hook.py index 323e8a639e6d..0e7a0ee9409c 100644 --- a/test-data/unit/plugins/type_anal_hook.py +++ b/test-data/unit/plugins/type_anal_hook.py @@ -1,37 +1,36 @@ -from typing import Optional +from typing import Optional, Callable -from mypy.plugin import Plugin, TypeAnalyzeHook, SemanticAnalysisPluginContext +from mypy.plugin import Plugin, AnalyzeTypeContext from mypy.types import Type, UnboundType, TypeList, AnyType, NoneTyp, CallableType class TypeAnalyzePlugin(Plugin): - def get_type_analyze_hook(self, fullname: str) -> Optional[TypeAnalyzeHook]: + def get_type_analyze_hook(self, fullname: str + ) -> Optional[Callable[[AnalyzeTypeContext], Type]]: if fullname == 'm.Signal': return signal_type_analyze_callback return None -def signal_type_analyze_callback( - typ: UnboundType, - context: SemanticAnalysisPluginContext) -> Type: - if (len(typ.args) != 1 - or not isinstance(typ.args[0], TypeList)): - context.fail('Invalid "Signal" type (expected "Signal[[t, ...]]")', context.context) +def signal_type_analyze_callback(ctx: AnalyzeTypeContext) -> Type: + if (len(ctx.type.args) != 1 + or not isinstance(ctx.type.args[0], TypeList)): + ctx.api.fail('Invalid "Signal" type (expected "Signal[[t, ...]]")', ctx.context) return AnyType() - args = typ.args[0] + args = ctx.type.args[0] assert isinstance(args, TypeList) - analyzed = context.analyze_arg_list(args) + analyzed = ctx.api.analyze_callable_args(args) if analyzed is None: return AnyType() # Error generated elsewhere arg_types, arg_kinds, arg_names = analyzed - arg_types = [context.analyze_type(arg) for arg in arg_types] + arg_types = [ctx.api.analyze_type(arg) for arg in arg_types] type_arg = CallableType(arg_types, arg_kinds, arg_names, NoneTyp(), - context.named_instance('builtins.function', [])) - return context.named_instance('m.Signal', [type_arg]) + ctx.api.named_type('builtins.function', [])) + return ctx.api.named_type('m.Signal', [type_arg]) def plugin(version): From 9b3322dd8ea6ead00aba18c4a4a5a4dabd776241 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Jun 2017 17:11:27 +0100 Subject: [PATCH 06/10] Minor tweaks --- mypy/plugin.py | 4 ++-- mypy/semanal.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mypy/plugin.py b/mypy/plugin.py index 848d12233551..7fb079bb9812 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -13,7 +13,7 @@ class AnalyzerPluginInterface: - """Interface for accessing semantic analyzer functionality from plugins.""" + """Interface for accessing semantic analyzer functionality in plugins.""" @abstractmethod def fail(self, msg: str, ctx: Context) -> None: @@ -43,7 +43,7 @@ def analyze_callable_args(self, args: TypeList) -> Optional[Tuple[List[Type], class CheckerPluginInterface: - """Interface for accessing type checker functionality from plugins.""" + """Interface for accessing type checker functionality in plugins.""" msg = None # type: MessageBuilder diff --git a/mypy/semanal.py b/mypy/semanal.py index 111bb3b498c2..3e6237ffec7d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1540,7 +1540,8 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: self.lookup_fully_qualified, self.tvar_scope, self.fail, - self.plugin, allow_unnormalized=True) + self.plugin, + allow_unnormalized=True) if res and (not isinstance(res, Instance) or res.args): # TODO: What if this gets reassigned? name = s.lvalues[0] @@ -3106,7 +3107,8 @@ def visit_index_expr(self, expr: IndexExpr) -> None: self.lookup_fully_qualified, self.tvar_scope, self.fail, - self.plugin, allow_unnormalized=self.is_stub_file) + self.plugin, + allow_unnormalized=self.is_stub_file) expr.analyzed = TypeAliasExpr(res, fallback=self.alias_fallback(res), in_runtime=True) elif refers_to_class_or_function(expr.base): From 01fd4f43e5dafe2a4d370515ac8ca3db811e2b64 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Jun 2017 17:17:04 +0100 Subject: [PATCH 07/10] More tweaks --- mypy/plugin.py | 6 +++--- mypy/typeanal.py | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/mypy/plugin.py b/mypy/plugin.py index 7fb079bb9812..73653459bf83 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -28,9 +28,9 @@ def analyze_type(self, typ: Type) -> Type: raise NotImplementedError @abstractmethod - def analyze_callable_args(self, args: TypeList) -> Optional[Tuple[List[Type], - List[int], - List[Optional[str]]]]: + def analyze_callable_args(self, arglist: TypeList) -> Optional[Tuple[List[Type], + List[int], + List[Optional[str]]]]: raise NotImplementedError diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 2b37e531e340..4dc6dc1bd02f 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -433,13 +433,13 @@ def analyze_callable_type(self, t: UnboundType) -> Type: assert isinstance(ret, CallableType) return ret.accept(self) - def analyze_callable_args(self, t: TypeList) -> Optional[Tuple[List[Type], - List[int], - List[Optional[str]]]]: + def analyze_callable_args(self, arglist: TypeList) -> Optional[Tuple[List[Type], + List[int], + List[Optional[str]]]]: args = [] # type: List[Type] kinds = [] # type: List[int] names = [] # type: List[str] - for arg in t.items: + for arg in arglist.items: if isinstance(arg, CallableArgument): args.append(arg.typ) names.append(arg.name) @@ -464,8 +464,9 @@ def analyze_callable_args(self, t: TypeList) -> Optional[Tuple[List[Type], args.append(arg) kinds.append(ARG_POS) names.append(None) - check_arg_names(names, [t] * len(args), self.fail, "Callable") - check_arg_kinds(kinds, [t] * len(args), self.fail) + # Note that arglist below is only used for error context. + check_arg_names(names, [arglist] * len(args), self.fail, "Callable") + check_arg_kinds(kinds, [arglist] * len(args), self.fail) return args, kinds, names def analyze_type(self, t: Type) -> Type: From 6aa789c3a99919817d3e7338e3d9512c70824d08 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 Jun 2017 17:25:33 +0100 Subject: [PATCH 08/10] Fix error caused by merge --- mypy/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index cc0793e5c465..251d03df351e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -344,6 +344,7 @@ def load_plugins(options: Options, errors: Errors) -> Plugin: at least include the default plugin. """ + default_plugin = DefaultPlugin(options) # type: Plugin if not options.config_file: return default_plugin @@ -355,7 +356,7 @@ def plugin_error(message: str) -> None: errors.report(line, 0, message) errors.raise_error() - plugins = [DefaultPlugin(options)] # type: List[Plugin] + plugins = [default_plugin] errors.set_file(options.config_file, None) for plugin_path in options.plugins: # Plugin paths are relative to the config file location. From da74c587ad30418275857ae9e57c53a5a7a222b7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jun 2017 14:33:51 +0100 Subject: [PATCH 09/10] Address review feedback --- mypy/build.py | 13 +++++-------- mypy/checkexpr.py | 7 +++++++ test-data/unit/check-custom-plugin.test | 2 +- test-data/unit/plugins/named_callable.py | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 251d03df351e..defe2fb38914 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -341,7 +341,7 @@ def load_plugins(options: Options, errors: Errors) -> Plugin: """Load all configured plugins. Return a plugin that encapsulates all plugins chained together. Always - at least include the default plugin. + at least include the default plugin (it's last in the chain). """ default_plugin = DefaultPlugin(options) # type: Plugin @@ -356,7 +356,7 @@ def plugin_error(message: str) -> None: errors.report(line, 0, message) errors.raise_error() - plugins = [default_plugin] + custom_plugins = [] # type: List[Plugin] errors.set_file(options.config_file, None) for plugin_path in options.plugins: # Plugin paths are relative to the config file location. @@ -396,15 +396,12 @@ def plugin_error(message: str) -> None: 'Return value of "plugin" must be a subclass of "mypy.plugin.Plugin" ' '(in {})'.format(plugin_path)) try: - plugins.append(plugin_type(options)) + custom_plugins.append(plugin_type(options)) except Exception: print('Error constructing plugin instance of {}\n'.format(plugin_type.__name__)) raise # Propagate to display traceback - if len(plugins) == 1: - return plugins[0] - else: - # Custom plugins take precendence over built-in plugins. - return ChainedPlugin(options, plugins) + # Custom plugins take precedence over the default plugin. + return ChainedPlugin(options, custom_plugins + [default_plugin]) def find_config_file_line_number(path: str, section: str, setting_name: str) -> int: diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index ae2a7fea309c..3cdc3006d827 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -380,6 +380,13 @@ def apply_function_plugin(self, context: Context) -> Type: """Use special case logic to infer the return type of a specific named function/method. + Caller must ensure that a plugin hook exists. There are two different cases: + + - If object_type is None, the caller must ensure that a function hook exists + for fullname. + - If object_type is not None, the caller must ensure that a method hook exists + for fullname. + Return the inferred return type. """ formal_arg_types = [[] for _ in range(num_formals)] # type: List[List[Type]] diff --git a/test-data/unit/check-custom-plugin.test b/test-data/unit/check-custom-plugin.test index 7e4ecf8d29f1..d1fdd9fbcfa9 100644 --- a/test-data/unit/check-custom-plugin.test +++ b/test-data/unit/check-custom-plugin.test @@ -95,7 +95,7 @@ s('') # E: Argument 1 has incompatible type "str"; expected "int" from typing import TypeVar, Generic, Callable T = TypeVar('T', bound=Callable[..., None]) class Signal(Generic[T]): - __call__: Callable[..., None] + __call__: Callable[..., None] # This type is replaced by the plugin [file mypy.ini] [[mypy] plugins=/test-data/unit/plugins/attrhook.py diff --git a/test-data/unit/plugins/named_callable.py b/test-data/unit/plugins/named_callable.py index 51ed7239262f..9acff69c1908 100644 --- a/test-data/unit/plugins/named_callable.py +++ b/test-data/unit/plugins/named_callable.py @@ -6,7 +6,7 @@ class MyPlugin(Plugin): def get_function_hook(self, fullname): if fullname == 'm.decorator1': return decorator_call_hook - if fullname == 'm._decorated': + if fullname == 'm._decorated': # This is a dummy name generated by the plugin return decorate_hook return None From 698bdefd5f562f9643682897e0c7ceea1022f313 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jun 2017 14:40:09 +0100 Subject: [PATCH 10/10] Minor refactoring --- mypy/plugin.py | 30 ++++++++++++------------ test-data/unit/plugins/named_callable.py | 12 +++++----- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/mypy/plugin.py b/mypy/plugin.py index 73653459bf83..f94790a06e96 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -60,7 +60,7 @@ def named_generic_type(self, name: str, args: List[Type]) -> Instance: FunctionContext = NamedTuple( 'FunctionContext', [ ('arg_types', List[List[Type]]), # List of actual caller types for each formal argument - ('inferred_return_type', Type), # Return type inferred from signature + ('default_return_type', Type), # Return type inferred from signature ('args', List[List[Expression]]), # Actual expressions for each formal argument ('context', Context), ('api', CheckerPluginInterface)]) @@ -70,9 +70,9 @@ def named_generic_type(self, name: str, args: List[Type]) -> Instance: # you have to use a method hook instead. MethodSigContext = NamedTuple( 'MethodSigContext', [ - ('type', Type), # Base object type for method call - ('args', List[List[Expression]]), # Actual expressions for each formal argument - ('signature', CallableType), # Original signature of the method + ('type', Type), # Base object type for method call + ('args', List[List[Expression]]), # Actual expressions for each formal argument + ('default_signature', CallableType), # Original signature of the method ('context', Context), ('api', CheckerPluginInterface)]) @@ -84,7 +84,7 @@ def named_generic_type(self, name: str, args: List[Type]) -> Instance: 'MethodContext', [ ('type', Type), # Base object type for method call ('arg_types', List[List[Type]]), - ('inferred_return_type', Type), + ('default_return_type', Type), ('args', List[List[Expression]]), ('context', Context), ('api', CheckerPluginInterface)]) @@ -93,7 +93,7 @@ def named_generic_type(self, name: str, args: List[Type]) -> Instance: AttributeContext = NamedTuple( 'AttributeContext', [ ('type', Type), # Type of object with attribute - ('inferred_attr_type', Type), # Original attribute type + ('default_attr_type', Type), # Original attribute type ('context', Context), ('api', CheckerPluginInterface)]) @@ -227,12 +227,12 @@ def open_callback(ctx: FunctionContext) -> Type: elif isinstance(ctx.args[1][0], StrExpr): mode = ctx.args[1][0].value if mode is not None: - assert isinstance(ctx.inferred_return_type, Instance) + assert isinstance(ctx.default_return_type, Instance) if 'b' in mode: return ctx.api.named_generic_type('typing.BinaryIO', []) else: return ctx.api.named_generic_type('typing.TextIO', []) - return ctx.inferred_return_type + return ctx.default_return_type def contextmanager_callback(ctx: FunctionContext) -> Type: @@ -241,14 +241,14 @@ def contextmanager_callback(ctx: FunctionContext) -> Type: if ctx.arg_types and len(ctx.arg_types[0]) == 1: arg_type = ctx.arg_types[0][0] if (isinstance(arg_type, CallableType) - and isinstance(ctx.inferred_return_type, CallableType)): + and isinstance(ctx.default_return_type, CallableType)): # The stub signature doesn't preserve information about arguments so # add them back here. - return ctx.inferred_return_type.copy_modified( + return ctx.default_return_type.copy_modified( arg_types=arg_type.arg_types, arg_kinds=arg_type.arg_kinds, arg_names=arg_type.arg_names) - return ctx.inferred_return_type + return ctx.default_return_type def typed_dict_get_signature_callback(ctx: MethodSigContext) -> CallableType: @@ -257,7 +257,7 @@ def typed_dict_get_signature_callback(ctx: MethodSigContext) -> CallableType: This is used to get better type context for the second argument that depends on a TypedDict value type. """ - signature = ctx.signature + signature = ctx.default_signature if (isinstance(ctx.type, TypedDictType) and len(ctx.args) == 2 and len(ctx.args[0]) == 1 @@ -293,7 +293,7 @@ def typed_dict_get_callback(ctx: MethodContext) -> Type: else: ctx.api.msg.typeddict_item_name_not_found(ctx.type, key, ctx.context) return AnyType() - return ctx.inferred_return_type + return ctx.default_return_type def int_pow_callback(ctx: MethodContext) -> Type: @@ -307,9 +307,9 @@ def int_pow_callback(ctx: MethodContext) -> Type: exponent = -arg.expr.value else: # Right operand not an int literal or a negated literal -- give up. - return ctx.inferred_return_type + return ctx.default_return_type if exponent >= 0: return ctx.api.named_generic_type('builtins.int', []) else: return ctx.api.named_generic_type('builtins.float', []) - return ctx.inferred_return_type + return ctx.default_return_type diff --git a/test-data/unit/plugins/named_callable.py b/test-data/unit/plugins/named_callable.py index 9acff69c1908..e40d181d2bad 100644 --- a/test-data/unit/plugins/named_callable.py +++ b/test-data/unit/plugins/named_callable.py @@ -12,16 +12,16 @@ def get_function_hook(self, fullname): def decorator_call_hook(ctx): - if isinstance(ctx.inferred_return_type, CallableType): - return ctx.inferred_return_type.copy_modified(name='m._decorated') - return ctx.inferred_return_type + if isinstance(ctx.default_return_type, CallableType): + return ctx.default_return_type.copy_modified(name='m._decorated') + return ctx.default_return_type def decorate_hook(ctx): - if isinstance(ctx.inferred_return_type, CallableType): - return ctx.inferred_return_type.copy_modified( + if isinstance(ctx.default_return_type, CallableType): + return ctx.default_return_type.copy_modified( ret_type=ctx.api.named_generic_type('builtins.str', [])) - return ctx.inferred_return_type + return ctx.default_return_type def plugin(version):