[3.11] gh-115421: List all test/ subdirs in Makefile, and test them (G…

…H-115813)

This backports:
- GH-115813
- GH-115422

Unlike on the main branch, new directories are added to the end,
so they're a bit easier to patch out if a redistributor needs to do so.

On main & 3.12, there's a special case for `idlelib/idle_test`; on
3.11 TESTSUBDIRS has several more entries that are not in `test/`.
This backport ignores all of them (including idlelib).
(The alternative would be list them, as additions to TEST_HOME_DIR.
But that's probably too invasive; people might split stdlib up in
surprising ways.)

Co-authored-by: Nikita Sobolev <[email protected]>

Lib/test/test_tools/test_makefile.py

@@ -0,0 +1,67 @@
+"""
+Tests for `Makefile`.
+"""
+
+import os
+import unittest
+from test import support
+import sysconfig
+
+MAKEFILE = sysconfig.get_makefile_filename()
+
+if not support.check_impl_detail(cpython=True):
+    raise unittest.SkipTest('cpython only')
+if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE):
+    raise unittest.SkipTest('Makefile could not be found')
+
+
+class TestMakefile(unittest.TestCase):
+    def list_test_dirs(self):
+        result = []
+        found_testsubdirs = False
+        with open(MAKEFILE, 'r', encoding='utf-8') as f:
+            for line in f:
+                if line.startswith('TESTSUBDIRS='):
+                    found_testsubdirs = True
+                    result.append(
+                        line.removeprefix('TESTSUBDIRS=').replace(
+                            '\\', '',
+                        ).strip(),
+                    )
+                    continue
+                if found_testsubdirs:
+                    if '\t' not in line:
+                        break
+                    result.append(line.replace('\\', '').strip())
+
+        # In Python 3.11 (and lower), many test modules are not in
+        # the tests/ directory. This check ignores them.
+        result = [d for d in result if d.startswith('test/') or d == 'test']
+
+        return result
+
+    def test_makefile_test_folders(self):
+        test_dirs = self.list_test_dirs()
+
+        used = []
+        for dirpath, _, _ in os.walk(support.TEST_HOME_DIR):
+            dirname = os.path.basename(dirpath)
+            if dirname == '__pycache__':
+                continue
+
+            relpath = os.path.relpath(dirpath, support.STDLIB_DIR)
+            with self.subTest(relpath=relpath):
+                self.assertIn(
+                    relpath,
+                    test_dirs,
+                    msg=(
+                        f"{relpath!r} is not included in the Makefile's list "
+                        "of test directories to install"
+                    )
+                )
+                used.append(relpath)
+
+        # Check that there are no extra entries:
+        unique_test_dirs = set(test_dirs)
+        self.assertSetEqual(unique_test_dirs, set(used))
+        self.assertEqual(len(test_dirs), len(unique_test_dirs))

Makefile.pre.in

@@ -2065,7 +2065,11 @@ TESTSUBDIRS=	ctypes/test \
 		tkinter/test/test_tkinter \
 		tkinter/test/test_ttk \
 		unittest/test \
-		unittest/test/testmock
+		unittest/test/testmock \
+		test/test_concurrent_futures \
+		test/test_multiprocessing_fork \
+		test/test_multiprocessing_forkserver \
+		test/test_multiprocessing_spawn
 
 TEST_MODULES=@TEST_MODULES@
 libinstall:	all $(srcdir)/Modules/xxmodule.c