Run PEP 561 tests in parallel but use lock around pip install steps

.gitignore

@@ -14,6 +14,7 @@ venv/
 .mypy_cache/
 .incremental_checker_cache.json
 .cache
+test-data/packages/.pip_lock
 dmypy.json
 .dmypy.json
 

mypy/test/config.py

@@ -15,3 +15,12 @@
 # This is *within* the tempfile.TemporaryDirectory that is chroot'ed per testcase.
 # It is also hard-coded in numerous places, so don't change it.
 test_temp_dir = 'tmp'
+
+# The PEP 561 tests do a bunch of pip installs which, even though they operate
+# on distinct temporary virtual environments, run into race conditions on shared
+# file-system state. To make this work reliably in parallel mode, we'll use a
+# FileLock courtesy of the tox-dev/py-filelock package.
+# Ref. https://github.com/python/mypy/issues/12615
+# Ref. mypy/test/testpep561.py
+pip_lock = os.path.join(package_path, '.pip_lock')
+pip_timeout = 60

mypy/test/testpep561.py

@@ -1,4 +1,5 @@
 from contextlib import contextmanager
+import filelock
 import os
 import pytest
 import re
@@ -9,7 +10,7 @@
 from typing import Tuple, List, Generator
 
 import mypy.api
-from mypy.test.config import package_path
+from mypy.test.config import package_path, pip_lock, pip_timeout
 from mypy.util import try_find_python2_interpreter
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.config import test_temp_dir
@@ -77,11 +78,15 @@ def install_package(pkg: str,
         env = {'PIP_BUILD': dir}
         # Inherit environment for Windows
         env.update(os.environ)
-        proc = subprocess.run(install_cmd,
-                              cwd=working_dir,
-                              stdout=PIPE,
-                              stderr=PIPE,
-                              env=env)
+        try:
+            with filelock.FileLock(pip_lock, timeout=pip_timeout):
+                proc = subprocess.run(install_cmd,
+                                      cwd=working_dir,
+                                      stdout=PIPE,
+                                      stderr=PIPE,
+                                      env=env)
+        except filelock.Timeout as err:
+            raise Exception("Failed to acquire {}".format(pip_lock)) from err
     if proc.returncode != 0:
         raise Exception(proc.stdout.decode('utf-8') + proc.stderr.decode('utf-8'))
 

runtests.py

@@ -58,11 +58,15 @@
     'pytest-slow': ['pytest', '-q', '-k', ' or '.join(
         [SAMPLES,
          TYPESHED,
-         PEP561,
          DAEMON,
          MYPYC_EXTERNAL,
          MYPYC_COMMAND_LINE,
          ERROR_STREAM])],
+
+    # Test cases that might take minutes to run
+    'pytest-slower': ['pytest', '-q', '-k', ' or '.join(
+        [PEP561])],
+
     # Test cases to run in typeshed CI
     'typeshed-ci': ['pytest', '-q', '-k', ' or '.join([CMDLINE,
                                                        EVALUATION,

test-requirements.txt

@@ -1,6 +1,8 @@
 -r mypy-requirements.txt
 -r build-requirements.txt
 attrs>=18.0
+filelock>=3.0.0,<3.4.2; python_version<'3.7'
+filelock>=3.0.0; python_version>='3.7'
 flake8==3.9.2
 flake8-bugbear==22.3.20
 flake8-pyi>=20.5