pythongh-108388: regrtest runs slowest tests first

The Python test suite (regrtest) now runs the 20 slowest tests first and
then other tests, to better use all available CPUs when running tests in
parallel.

Lib/test/libregrtest/main.py

@@ -12,7 +12,7 @@
 from test.libregrtest.cmdline import _parse_args
 from test.libregrtest.runtest import (
     findtests, runtest, get_abs_module, is_failed,
-    STDTESTS, NOTTESTS, PROGRESS_MIN_TIME,
+    PROGRESS_MIN_TIME,
     Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted,
     ChildError, DidNotRun)
 from test.libregrtest.setup import setup_tests
@@ -42,6 +42,32 @@
 EXITCODE_ENV_CHANGED = 3
 EXITCODE_NO_TESTS_RAN = 4
 
+# Coarse heuristic: tests taking at least 1 minute on a modern
+# developer laptop. The list should have less than 20 tests.
+SLOWEST_TESTS = frozenset((
+    # more or less sorted from the slowest to the fastest
+    "test_concurrent_futures",
+    "test_multiprocessing_spawn",
+    "test_multiprocessing_forkserver",
+    "test_multiprocessing_fork",
+    "test_multiprocessing_main_handling",
+    "test_pickle",
+    "test_compileall",
+    "test_cppext",
+    "test_venv",
+    "test_gdb",
+    "test_tools",
+    "test_peg_generator",
+    "test_perf_profiler",
+    "test_buffer",
+    "test_subprocess",
+    "test_signal",
+    "test_tarfile",
+    "test_regrtest",
+    "test_socket",
+    "test_io",
+))
+
 
 class Regrtest:
     """Execute a test suite.
@@ -246,21 +272,18 @@ def find_tests(self, tests):
             # add default PGO tests if no tests are specified
             setup_pgo_tests(self.ns)
 
-        stdtests = STDTESTS[:]
-        nottests = NOTTESTS.copy()
+        exclude_tests = set()
         if self.ns.exclude:
             for arg in self.ns.args:
-                if arg in stdtests:
-                    stdtests.remove(arg)
-                nottests.add(arg)
+                exclude_tests.add(arg)
             self.ns.args = []
 
         # if testdir is set, then we are not running the python tests suite, so
         # don't add default tests to be executed or skipped (pass empty values)
         if self.ns.testdir:
-            alltests = findtests(self.ns.testdir, list(), set())
+            alltests = findtests(self.ns.testdir)
         else:
-            alltests = findtests(self.ns.testdir, stdtests, nottests)
+            alltests = findtests(self.ns.testdir, exclude=exclude_tests)
 
         if not self.ns.fromfile:
             self.selected = self.tests or self.ns.args or alltests
@@ -282,11 +305,31 @@ def find_tests(self, tests):
                 print("Couldn't find starting test (%s), using all tests"
                       % self.ns.start, file=sys.stderr)
 
+        self.group_randomize_tests()
+
+    def group_randomize_tests(self):
         if self.ns.randomize:
             if self.ns.random_seed is None:
                 self.ns.random_seed = random.randrange(10000000)
             random.seed(self.ns.random_seed)
-            random.shuffle(self.selected)
+
+        # group slow tests
+        slow = []
+        other = []
+        for name in self.selected:
+            if name in SLOWEST_TESTS:
+                slow.append(name)
+            else:
+                other.append(name)
+
+        if self.ns.randomize:
+            if slow:
+                random.shuffle(slow)
+            if other:
+                random.shuffle(other)
+
+        # gh-108388: Run the slowest first, and then other tests
+        self.selected = slow + other
 
     def list_tests(self):
         for name in self.selected:

Lib/test/libregrtest/runtest.py

@@ -125,24 +125,6 @@ def __str__(self) -> str:
 # the test is running in background
 PROGRESS_MIN_TIME = 30.0   # seconds
 
-# small set of tests to determine if we have a basically functioning interpreter
-# (i.e. if any of these fail, then anything else is likely to follow)
-STDTESTS = [
-    'test_grammar',
-    'test_opcodes',
-    'test_dict',
-    'test_builtin',
-    'test_exceptions',
-    'test_types',
-    'test_unittest',
-    'test_doctest',
-    'test_doctest2',
-    'test_support'
-]
-
-# set of tests that we don't want to be executed when using regrtest
-NOTTESTS = set()
-
 #If these test directories are encountered recurse into them and treat each
 # test_ .py or dir as a separate test module. This can increase parallelism.
 # Beware this can't generally be done for any directory with sub-tests as the
@@ -166,22 +148,23 @@ def findtestdir(path=None):
     return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir
 
 
-def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, *, split_test_dirs=SPLITTESTDIRS, base_mod=""):
+def findtests(testdir=None, *, exclude=(), split_test_dirs=SPLITTESTDIRS, base_mod=""):
     """Return a list of all applicable test modules."""
     testdir = findtestdir(testdir)
     names = os.listdir(testdir)
     tests = []
-    others = set(stdtests) | nottests
     for name in names:
         mod, ext = os.path.splitext(name)
-        if mod[:5] == "test_" and mod not in others:
-            if mod in split_test_dirs:
-                subdir = os.path.join(testdir, mod)
-                mod = f"{base_mod or 'test'}.{mod}"
-                tests.extend(findtests(subdir, [], nottests, split_test_dirs=split_test_dirs, base_mod=mod))
-            elif ext in (".py", ""):
-                tests.append(f"{base_mod}.{mod}" if base_mod else mod)
-    return stdtests + sorted(tests)
+        if not mod.startswith("test_") or mod in exclude:
+            continue
+
+        if mod in split_test_dirs:
+            subdir = os.path.join(testdir, mod)
+            mod = f"{base_mod or 'test'}.{mod}"
+            tests.extend(findtests(subdir, exclude=exclude, split_test_dirs=split_test_dirs, base_mod=mod))
+        elif ext in (".py", ""):
+            tests.append(f"{base_mod}.{mod}" if base_mod else mod)
+    return sorted(tests)
 
 
 def get_abs_module(ns: Namespace, test_name: str) -> str:

Misc/NEWS.d/next/Tests/2023-08-24-02-36-15.gh-issue-108388.Z08JFZ.rst

@@ -0,0 +1,3 @@
+The Python test suite (regrtest) now runs the 20 slowest tests first and
+then other tests, to better use all available CPUs when running tests in
+parallel. Patch Victor Stinner.