From 02d81aefa06fcf58cfd125a7a760039b4f892319 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Mon, 5 Mar 2018 11:35:53 -0800 Subject: [PATCH] New files shouldn't trigger a coarse-grained rebuild in fg cache mode (#4669) --- mypy/build.py | 19 +++- mypy/server/update.py | 1 + mypy/test/testfinegrained.py | 17 +++- test-data/unit/fine-grained-modules.test | 118 +++++++++++++++++++++++ 4 files changed, 151 insertions(+), 4 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index b6a4b6325618..745d682ee0b3 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -588,6 +588,7 @@ def __init__(self, data_dir: str, self.data_dir = data_dir self.errors = errors self.errors.set_ignore_prefix(ignore_prefix) + self.only_load_from_cache = options.use_fine_grained_cache self.lib_path = tuple(lib_path) self.source_set = source_set self.reports = reports @@ -1677,6 +1678,13 @@ def __init__(self, for id, line in zip(self.meta.dependencies, self.meta.dep_lines)} self.child_modules = set(self.meta.child_modules) else: + # In fine-grained cache mode, pretend we only know about modules that + # have cache information and defer handling new modules until the + # fine-grained update. + if manager.only_load_from_cache: + manager.log("Deferring module to fine-grained update %s (%s)" % (path, id)) + raise ModuleNotFound + # Parse the file (and then some) to get the dependencies. self.parse_file() self.compute_dependencies() @@ -2096,6 +2104,15 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph: manager.log("Mypy version %s" % __version__) t0 = time.time() graph = load_graph(sources, manager) + + # This is a kind of unfortunate hack to work around some of fine-grained's + # fragility: if we have loaded less than 50% of the specified files from + # cache in fine-grained cache mode, load the graph again honestly. + if manager.options.use_fine_grained_cache and len(graph) < 0.50 * len(sources): + manager.log("Redoing load_graph because too much was missing") + manager.only_load_from_cache = False + graph = load_graph(sources, manager) + t1 = time.time() manager.add_stats(graph_size=len(graph), stubs_found=sum(g.path is not None and g.path.endswith('.pyi') @@ -2209,7 +2226,7 @@ def load_graph(sources: List[BuildSource], manager: BuildManager, there are syntax errors. """ - graph = old_graph or {} # type: Graph + graph = old_graph if old_graph is not None else {} # type: Graph # The deque is used to implement breadth-first traversal. # TODO: Consider whether to go depth-first instead. This may diff --git a/mypy/server/update.py b/mypy/server/update.py index a5ff3370d64e..7795d5c1813a 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -174,6 +174,7 @@ def __init__(self, # this directly reflected in load_graph's interface. self.options.cache_dir = os.devnull manager.saved_cache = {} + manager.only_load_from_cache = False # Active triggers during the last update self.triggered = [] # type: List[str] diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index f102d6571da2..54ba13b200fd 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -75,7 +75,8 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: return main_src = '\n'.join(testcase.input) - sources_override = self.parse_sources(main_src) + step = 1 + sources_override = self.parse_sources(main_src, step) messages, manager, graph = self.build(main_src, testcase, sources_override, build_cache=self.use_cache, enable_cache=self.use_cache) @@ -92,6 +93,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: steps = testcase.find_steps() all_triggered = [] for operations in steps: + step += 1 modules = [] for op in operations: if isinstance(op, UpdateFile): @@ -102,6 +104,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: # Delete file os.remove(op.path) modules.append((op.module, op.path)) + sources_override = self.parse_sources(main_src, step) if sources_override is not None: modules = [(module, path) for module, path in sources_override @@ -181,14 +184,22 @@ def format_triggered(self, triggered: List[List[str]]) -> List[str]: result.append(('%d: %s' % (n + 2, ', '.join(filtered))).strip()) return result - def parse_sources(self, program_text: str) -> Optional[List[Tuple[str, str]]]: + def parse_sources(self, program_text: str, + incremental_step: int) -> Optional[List[Tuple[str, str]]]: """Return target (module, path) tuples for a test case, if not using the defaults. These are defined through a comment like '# cmd: main a.py' in the test case description. """ - # TODO: Support defining separately for each incremental step. m = re.search('# cmd: mypy ([a-zA-Z0-9_./ ]+)$', program_text, flags=re.MULTILINE) + regex = '# cmd{}: mypy ([a-zA-Z0-9_./ ]+)$'.format(incremental_step) + alt_m = re.search(regex, program_text, flags=re.MULTILINE) + if alt_m is not None and incremental_step > 1: + # Optionally return a different command if in a later step + # of incremental mode, otherwise default to reusing the + # original cmd. + m = alt_m + if m: # The test case wants to use a non-default set of files. paths = m.group(1).strip().split() diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test index c765af39553c..3f0d47921f34 100644 --- a/test-data/unit/fine-grained-modules.test +++ b/test-data/unit/fine-grained-modules.test @@ -971,3 +971,121 @@ x = Foo() == == main:2: error: Too few arguments for "foo" of "Foo" + +-- This series of tests is designed to test adding a new module that +-- doesn't appear in the cache, for cache mode. They aren't run only +-- in cache mode, though, because they are still perfectly good +-- regular tests. +[case testAddModuleAfterCache1] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py +# cmd3: mypy main a.py b.py +import a +[file a.py] +pass +[file a.py.2] +import b +b.foo(0) +[file b.py.2] +def foo() -> None: pass +[file b.py.3] +def foo(x: int) -> None: pass +[out] +== +a.py:2: error: Too many arguments for "foo" +== + +[case testAddModuleAfterCache2] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py +# cmd3: mypy main a.py b.py +# flags: --ignore-missing-imports --follow-imports=skip +import a +[file a.py] +import b +b.foo(0) +[file b.py.2] +def foo() -> None: pass +[file b.py.3] +def foo(x: int) -> None: pass +[out] +== +a.py:2: error: Too many arguments for "foo" +== + +[case testAddModuleAfterCache3] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py c.py d.py e.py f.py g.py +# flags: --ignore-missing-imports --follow-imports=skip +import a +[file a.py] +import b, c, d, e, f, g +[file b.py.2] +[file c.py.2] +[file d.py.2] +[file e.py.2] +[file f.py.2] +[file g.py.2] +[out] +== + +[case testAddModuleAfterCache4] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py +# cmd3: mypy main a.py b.py +# flags: --ignore-missing-imports --follow-imports=skip +import a +import b +[file a.py] +def foo() -> None: pass +[file b.py.2] +import a +a.foo(10) +[file a.py.3] +def foo(x: int) -> None: pass +[out] +== +b.py:2: error: Too many arguments for "foo" +== + +[case testAddModuleAfterCache5] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py +# cmd3: mypy main a.py b.py +# flags: --ignore-missing-imports --follow-imports=skip +import a +import b +[file a.py] +def foo(x: int) -> None: pass +[file a.py.2] +def foo() -> None: pass +[file b.py.2] +import a +a.foo(10) +[file a.py.3] +def foo(x: int) -> None: pass +[out] +== +b.py:2: error: Too many arguments for "foo" +== + +[case testAddModuleAfterCache6] +# cmd: mypy main a.py +# cmd2: mypy main a.py b.py +# cmd3: mypy main a.py b.py +# flags: --ignore-missing-imports --follow-imports=skip +import a +[file a.py] +import b +b.foo() +[file a.py.2] +import b +b.foo(0) +[file b.py.2] +def foo() -> None: pass +[file b.py.3] +def foo(x: int) -> None: pass +[out] +== +a.py:2: error: Too many arguments for "foo" +==