From 4a6dd7add5e2bd8d6fc66391fe0e1d5cb8631fc0 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Tue, 16 Jan 2018 12:16:20 -0800 Subject: [PATCH 1/4] Compute fine-grained dependencies during normal operation and store them --- mypy/build.py | 29 +++++++++++++++++++++++++++-- mypy/main.py | 2 ++ mypy/options.py | 4 +++- mypy/server/update.py | 24 +++++++----------------- 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 301c18cef9f9..8c2bd5cd47b7 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -52,6 +52,7 @@ from mypy.version import __version__ from mypy.plugin import Plugin, DefaultPlugin, ChainedPlugin from mypy.defaults import PYTHON3_VERSION_MIN +from mypy.server.deps import get_dependencies PYTHON_EXTENSIONS = ['.pyi', '.py'] @@ -1183,6 +1184,7 @@ def compute_hash(text: str) -> str: def write_cache(id: str, path: str, tree: MypyFile, + fine_grained_deps: Dict[str, List[str]], dependencies: List[str], suppressed: List[str], child_modules: List[str], dep_prios: List[int], old_interface_hash: str, source_hash: str, @@ -1221,7 +1223,9 @@ def write_cache(id: str, path: str, tree: MypyFile, assert os.path.dirname(meta_json) == parent # Serialize data and analyze interface - data = tree.serialize() + data = {'tree': tree.serialize(), + 'fine_grained_deps': fine_grained_deps, + } if manager.options.debug_cache: data_str = json.dumps(data, indent=2, sort_keys=True) else: @@ -1523,6 +1527,8 @@ class State: # Whether the module has an error or any of its dependencies have one. transitive_error = False + fine_grained_deps = None # type: Dict[str, Set[str]] + # Type checker used for checking this file. Use type_checker() for # access and to construct this on demand. _type_checker = None # type: Optional[TypeChecker] @@ -1551,6 +1557,7 @@ def __init__(self, self.id = id or '__main__' self.options = manager.options.clone_for_module(self.id) self._type_checker = None + self.fine_grained_deps = {} if not path and source is None: assert id is not None file_id = id @@ -1734,7 +1741,9 @@ def load_tree(self) -> None: with open(self.meta.data_json) as f: data = json.load(f) # TODO: Assert data file wasn't changed. - self.tree = MypyFile.deserialize(data) + self.tree = MypyFile.deserialize(data['tree']) + self.fine_grained_deps = {k: set(v) for k, v in data['fine_grained_deps']} + self.manager.modules[self.id] = self.tree self.manager.add_stats(fresh_trees=1) @@ -1977,6 +1986,19 @@ def _patch_indirect_dependencies(self, elif dep not in self.suppressed and dep in self.manager.missing_modules: self.suppressed.append(dep) + def find_fine_grained_deps(self) -> None: + assert self.tree is not None + if '/typeshed/' in self.xpath or self.xpath.startswith('typeshed/'): + # We don't track changes to typeshed -- the assumption is that they are only changed + # as part of mypy updates, which will invalidate everything anyway. + # + # TODO: Not a reliable test, as we could have a package named typeshed. + # TODO: Consider relaxing this -- maybe allow some typeshed changes to be tracked. + return + self.fine_grained_deps = get_dependencies(target=self.tree, + type_map=self.type_map(), + python_version=self.options.python_version) + def valid_references(self) -> Set[str]: assert self.ancestors is not None valid_refs = set(self.dependencies + self.suppressed + self.ancestors) @@ -2003,6 +2025,7 @@ def write_cache(self) -> None: dep_prios = self.dependency_priorities() new_interface_hash, self.meta = write_cache( self.id, self.path, self.tree, + {k: list(v) for k, v in self.fine_grained_deps.items()}, list(self.dependencies), list(self.suppressed), list(self.child_modules), dep_prios, self.interface_hash, self.source_hash, self.ignore_all, self.manager) @@ -2534,6 +2557,8 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No graph[id].transitive_error = True for id in stale: graph[id].finish_passes() + if manager.options.cache_fine_grained: + graph[id].find_fine_grained_deps() graph[id].generate_unused_ignore_notes() manager.flush_errors(manager.errors.file_messages(graph[id].xpath), False) graph[id].write_cache() diff --git a/mypy/main.py b/mypy/main.py index b64dcccd3fa4..cd9b691f4bd8 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -320,6 +320,8 @@ def add_invertible_flag(flag: str, parser.add_argument('--cache-dir', action='store', metavar='DIR', help="store module cache info in the given folder in incremental mode " "(defaults to '{}')".format(defaults.CACHE_DIR)) + parser.add_argument('--cache-fine-grained', action='store_true', + help="included fine-grained dependency information in the cache") parser.add_argument('--skip-version-check', action='store_true', help="allow using cache written by older mypy version") add_invertible_flag('--strict-optional', default=False, strict_flag=True, diff --git a/mypy/options.py b/mypy/options.py index f05253d034fc..c62520075d34 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -42,7 +42,8 @@ class Options: "disallow_untyped_decorators", } - OPTIONS_AFFECTING_CACHE = ((PER_MODULE_OPTIONS | {"quick_and_dirty", "platform"}) + OPTIONS_AFFECTING_CACHE = ((PER_MODULE_OPTIONS | + {"quick_and_dirty", "platform", "cache_fine_grained"}) - {"debug_cache"}) def __init__(self) -> None: @@ -142,6 +143,7 @@ def __init__(self) -> None: self.quick_and_dirty = False self.skip_version_check = False self.fine_grained_incremental = False + self.cache_fine_grained = False # Paths of user plugins self.plugins = [] # type: List[str] diff --git a/mypy/server/update.py b/mypy/server/update.py index e322b7bef315..8eae132af318 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -280,7 +280,7 @@ def update_single(self, module: str, path: str) -> Tuple[List[str], if not trigger.endswith('__>')] print('triggered:', sorted(filtered)) self.triggered.extend(triggered | self.previous_targets_with_errors) - update_dependencies({module: tree}, self.deps, graph, self.options) + collect_dependencies({module: tree}, self.deps, graph) propagate_changes_using_dependencies(manager, graph, self.deps, triggered, {module}, self.previous_targets_with_errors) @@ -319,7 +319,7 @@ def get_all_dependencies(manager: BuildManager, graph: Dict[str, State], options: Options) -> Dict[str, Set[str]]: """Return the fine-grained dependency map for an entire build.""" deps = {} # type: Dict[str, Set[str]] - update_dependencies(manager.modules, deps, graph, options) + collect_dependencies(manager.modules, deps, graph) return deps @@ -644,24 +644,14 @@ def find_import_line(node: MypyFile, target: str) -> Optional[int]: return None -def update_dependencies(new_modules: Mapping[str, Optional[MypyFile]], - deps: Dict[str, Set[str]], - graph: Dict[str, State], - options: Options) -> None: +def collect_dependencies(new_modules: Mapping[str, Optional[MypyFile]], + deps: Dict[str, Set[str]], + graph: Dict[str, State]) -> None: for id, node in new_modules.items(): if node is None: continue - if '/typeshed/' in node.path or node.path.startswith('typeshed/'): - # We don't track changes to typeshed -- the assumption is that they are only changed - # as part of mypy updates, which will invalidate everything anyway. - # - # TODO: Not a reliable test, as we could have a package named typeshed. - # TODO: Consider relaxing this -- maybe allow some typeshed changes to be tracked. - continue - module_deps = get_dependencies(target=node, - type_map=graph[id].type_map(), - python_version=options.python_version) - for trigger, targets in module_deps.items(): + graph[id].find_fine_grained_deps() + for trigger, targets in graph[id].fine_grained_deps.items(): deps.setdefault(trigger, set()).update(targets) From c5898281f52e09822ed69d6fee9d117df4c0f4e4 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Wed, 31 Jan 2018 13:37:03 -0800 Subject: [PATCH 2/4] Fix typo and lint error --- mypy/build.py | 2 +- mypy/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 8c2bd5cd47b7..360234bf32a6 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1225,7 +1225,7 @@ def write_cache(id: str, path: str, tree: MypyFile, # Serialize data and analyze interface data = {'tree': tree.serialize(), 'fine_grained_deps': fine_grained_deps, - } + } if manager.options.debug_cache: data_str = json.dumps(data, indent=2, sort_keys=True) else: diff --git a/mypy/main.py b/mypy/main.py index cd9b691f4bd8..10d8d1251b77 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -321,7 +321,7 @@ def add_invertible_flag(flag: str, help="store module cache info in the given folder in incremental mode " "(defaults to '{}')".format(defaults.CACHE_DIR)) parser.add_argument('--cache-fine-grained', action='store_true', - help="included fine-grained dependency information in the cache") + help="include fine-grained dependency information in the cache") parser.add_argument('--skip-version-check', action='store_true', help="allow using cache written by older mypy version") add_invertible_flag('--strict-optional', default=False, strict_flag=True, From d12ddf8f3097e88e87b29f915804b27007e2b969 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Wed, 31 Jan 2018 15:27:38 -0800 Subject: [PATCH 3/4] Fix fg deps cache loading --- mypy/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index 360234bf32a6..417dcaf5865c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1742,7 +1742,7 @@ def load_tree(self) -> None: data = json.load(f) # TODO: Assert data file wasn't changed. self.tree = MypyFile.deserialize(data['tree']) - self.fine_grained_deps = {k: set(v) for k, v in data['fine_grained_deps']} + self.fine_grained_deps = {k: set(v) for k, v in data['fine_grained_deps'].items()} self.manager.modules[self.id] = self.tree self.manager.add_stats(fresh_trees=1) From ff6ad5db80e1af3dc1f84e367fe1429dcaa03077 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Wed, 31 Jan 2018 15:39:40 -0800 Subject: [PATCH 4/4] Some cleanups --- mypy/build.py | 8 ++++---- mypy/server/update.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 417dcaf5865c..8aa95b6079e6 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1184,7 +1184,7 @@ def compute_hash(text: str) -> str: def write_cache(id: str, path: str, tree: MypyFile, - fine_grained_deps: Dict[str, List[str]], + serialized_fine_grained_deps: Dict[str, List[str]], dependencies: List[str], suppressed: List[str], child_modules: List[str], dep_prios: List[int], old_interface_hash: str, source_hash: str, @@ -1224,7 +1224,7 @@ def write_cache(id: str, path: str, tree: MypyFile, # Serialize data and analyze interface data = {'tree': tree.serialize(), - 'fine_grained_deps': fine_grained_deps, + 'fine_grained_deps': serialized_fine_grained_deps, } if manager.options.debug_cache: data_str = json.dumps(data, indent=2, sort_keys=True) @@ -1986,7 +1986,7 @@ def _patch_indirect_dependencies(self, elif dep not in self.suppressed and dep in self.manager.missing_modules: self.suppressed.append(dep) - def find_fine_grained_deps(self) -> None: + def compute_fine_grained_deps(self) -> None: assert self.tree is not None if '/typeshed/' in self.xpath or self.xpath.startswith('typeshed/'): # We don't track changes to typeshed -- the assumption is that they are only changed @@ -2558,7 +2558,7 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No for id in stale: graph[id].finish_passes() if manager.options.cache_fine_grained: - graph[id].find_fine_grained_deps() + graph[id].compute_fine_grained_deps() graph[id].generate_unused_ignore_notes() manager.flush_errors(manager.errors.file_messages(graph[id].xpath), False) graph[id].write_cache() diff --git a/mypy/server/update.py b/mypy/server/update.py index 8eae132af318..0cb81b5dc2df 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -650,7 +650,7 @@ def collect_dependencies(new_modules: Mapping[str, Optional[MypyFile]], for id, node in new_modules.items(): if node is None: continue - graph[id].find_fine_grained_deps() + graph[id].compute_fine_grained_deps() for trigger, targets in graph[id].fine_grained_deps.items(): deps.setdefault(trigger, set()).update(targets)