Add more test facilities, fix a bug revealed by them

mypy/dmypy_server.py

@@ -273,9 +273,6 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
         self.fscache = FileSystemCache(self.options.python_version)
         self.fswatcher = FileSystemWatcher(self.fscache)
         self.update_sources(sources)
-        if not self.options.use_fine_grained_cache:
-            # Stores the initial state of sources as a side effect.
-            self.fswatcher.find_changed()
         try:
             result = mypy.build.build(sources=sources,
                                       options=self.options,
@@ -292,7 +289,6 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
         graph = result.graph
         self.fine_grained_manager = FineGrainedBuildManager(manager, graph)
         self.previous_sources = sources
-        self.fscache.flush()
 
         # If we are using the fine-grained cache, build hasn't actually done
         # the typechecking on the updated files yet.
@@ -310,14 +306,20 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
 
             # Run an update
             changed = self.find_changed(sources)
+
+            # Find anything that has had its dependency list change
             for state in self.fine_grained_manager.graph.values():
                 if not state.is_fresh():
                     assert state.path is not None
                     changed.append((state.id, state.path))
 
             if changed:
                 messages = self.fine_grained_manager.update(changed)
-            self.fscache.flush()
+        else:
+            # Stores the initial state of sources as a side effect.
+            self.fswatcher.find_changed()
+
+        self.fscache.flush()
 
         status = 1 if messages else 0
         self.previous_messages = messages[:]

mypy/server/update.py

@@ -174,8 +174,14 @@ def __init__(self,
         # for the cache.
         self.manager.cache_enabled = False
         manager.saved_cache = {}
+
+        # Some hints to the test suite about what is going on:
         # Active triggers during the last update
         self.triggered = []  # type: List[str]
+        # Modules passed to update during the last update
+        self.changed_modules = []  # type: List[Tuple[str, str]]
+        # Modules processed during the last update
+        self.updated_modules = []  # type: List[str]
 
     def update(self, changed_modules: List[Tuple[str, str]]) -> List[str]:
         """Update previous build result by processing changed modules.
@@ -196,10 +202,13 @@ def update(self, changed_modules: List[Tuple[str, str]]) -> List[str]:
         """
         assert changed_modules, 'No changed modules'
 
+        self.changed_modules = changed_modules
+
         # Reset global caches for the new build.
         find_module_clear_caches()
 
         self.triggered = []
+        self.updated_modules = []
         changed_modules = dedupe_modules(changed_modules + self.stale)
         initial_set = {id for id, _ in changed_modules}
         self.manager.log_fine_grained('==== update %s ====' % ', '.join(
@@ -256,6 +265,7 @@ def update_single(self, module: str, path: str) -> Tuple[List[str],
             - Whether there was a blocking error in the module
         """
         self.manager.log_fine_grained('--- update single %r ---' % module)
+        self.updated_modules.append(module)
 
         # TODO: If new module brings in other modules, we parse some files multiple times.
         manager = self.manager

mypy/test/helpers.py

@@ -5,7 +5,7 @@
 import time
 import shutil
 
-from typing import List, Dict, Tuple, Callable, Any, Optional
+from typing import List, Iterable, Dict, Tuple, Callable, Any, Optional
 
 from mypy import defaults
 from mypy.test.config import test_temp_dir
@@ -98,6 +98,21 @@ def assert_string_arrays_equal(expected: List[str], actual: List[str],
         raise AssertionError(msg)
 
 
+def assert_module_equivalence(name: str,
+                              expected: Optional[Iterable[str]], actual: Iterable[str]) -> None:
+    if expected is not None:
+        expected_normalized = sorted(expected)
+        actual_normalized = sorted(set(actual).difference({"__main__"}))
+        assert_string_arrays_equal(
+            expected_normalized,
+            actual_normalized,
+            ('Actual modules ({}) do not match expected modules ({}) '
+             'for "[{} ...]"').format(
+                 ', '.join(actual_normalized),
+                 ', '.join(expected_normalized),
+                 name))
+
+
 def update_testcase_output(testcase: DataDrivenTestCase, output: List[str]) -> None:
     assert testcase.old_cwd is not None, "test was not properly set up"
     testcase_path = os.path.join(testcase.old_cwd, testcase.file)

mypy/test/testcheck.py

@@ -11,7 +11,7 @@
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import (
-    assert_string_arrays_equal, normalize_error_messages,
+    assert_string_arrays_equal, normalize_error_messages, assert_module_equivalence,
     retry_on_error, update_testcase_output, parse_options,
     copy_and_fudge_mtime
 )
@@ -190,29 +190,15 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0)
                 self.verify_cache(module_data, a, res.manager)
             if incremental_step > 1:
                 suffix = '' if incremental_step == 2 else str(incremental_step - 1)
-                self.check_module_equivalence(
+                assert_module_equivalence(
                     'rechecked' + suffix,
                     testcase.expected_rechecked_modules.get(incremental_step - 1),
                     res.manager.rechecked_modules)
-                self.check_module_equivalence(
+                assert_module_equivalence(
                     'stale' + suffix,
                     testcase.expected_stale_modules.get(incremental_step - 1),
                     res.manager.stale_modules)
 
-    def check_module_equivalence(self, name: str,
-                                 expected: Optional[Set[str]], actual: Set[str]) -> None:
-        if expected is not None:
-            expected_normalized = sorted(expected)
-            actual_normalized = sorted(actual.difference({"__main__"}))
-            assert_string_arrays_equal(
-                expected_normalized,
-                actual_normalized,
-                ('Actual modules ({}) do not match expected modules ({}) '
-                 'for "[{} ...]"').format(
-                    ', '.join(actual_normalized),
-                    ', '.join(expected_normalized),
-                    name))
-
     def verify_cache(self, module_data: List[Tuple[str, str, str]], a: List[str],
                      manager: build.BuildManager) -> None:
         # There should be valid cache metadata for each module except

mypy/test/testfinegrained.py

@@ -10,7 +10,7 @@
 import os
 import re
 
-from typing import List, Tuple, Optional, cast
+from typing import List, Set, Tuple, Optional, cast
 
 from mypy import build
 from mypy.build import BuildManager, BuildSource, Graph
@@ -21,7 +21,9 @@
 from mypy.test.data import (
     DataDrivenTestCase, DataSuite, UpdateFile, module_from_path
 )
-from mypy.test.helpers import assert_string_arrays_equal, parse_options, copy_and_fudge_mtime
+from mypy.test.helpers import (
+    assert_string_arrays_equal, parse_options, copy_and_fudge_mtime, assert_module_equivalence,
+)
 from mypy.server.mergecheck import check_consistency
 from mypy.dmypy_server import Server
 from mypy.main import expand_dir
@@ -96,6 +98,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
 
         steps = testcase.find_steps()
         all_triggered = []
+
         for operations in steps:
             step += 1
             for op in operations:
@@ -108,10 +111,26 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             sources = self.parse_sources(main_src, step)
             new_messages = self.run_check(server, sources)
 
+            assert server.fine_grained_manager
+
+            updated, changed = [], []
             if server.fine_grained_manager:
                 if CHECK_CONSISTENCY:
                     check_consistency(server.fine_grained_manager)
                 all_triggered.append(server.fine_grained_manager.triggered)
+
+                updated = server.fine_grained_manager.updated_modules
+                changed = [mod for mod, file in server.fine_grained_manager.changed_modules]
+
+            assert_module_equivalence(
+                'stale' + str(step - 1),
+                testcase.expected_stale_modules.get(step - 1),
+                changed)
+            assert_module_equivalence(
+                'rechecked' + str(step - 1),
+                testcase.expected_rechecked_modules.get(step - 1),
+                updated)
+
             new_messages = normalize_messages(new_messages)
 
             a.append('==')