From e436dfe21913e175ea65355818d5b81440240ad4 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Sun, 5 May 2024 16:28:37 +0200 Subject: [PATCH] Unify test directory handling (#11864) Previously, handling of test directories (`@tests` and `test_cases`) was distributed over multiple files and redundant. This unifies the handling in the `utils` module. This also fixes some instances where "package" was used instead of "distribution". And in a few instances paths were joined by using a `/` in a string, which is incompatible with Windows. Also move `runtests.py` from `scripts` to `tests`. This is required so that we can import `utils`, but it's also arguably the better fit. The only mention of the script is actually in the `tests/README.md` file. Helps with #11762. --- tests/README.md | 4 +-- tests/check_typeshed_structure.py | 13 +++++--- tests/regr_test.py | 44 +++++++++++++------------- {scripts => tests}/runtests.py | 11 ++++--- tests/stubtest_third_party.py | 8 ++--- tests/utils.py | 52 +++++++++++++++++++++++-------- 6 files changed, 81 insertions(+), 51 deletions(-) rename {scripts => tests}/runtests.py (96%) diff --git a/tests/README.md b/tests/README.md index 9826e9a58790..2e4e261d9aba 100644 --- a/tests/README.md +++ b/tests/README.md @@ -36,7 +36,7 @@ You can list or install all of a stubs package's external dependencies using the Run using: ```bash -(.venv3)$ python3 scripts/runtests.py / +(.venv3)$ python3 tests/runtests.py / ``` This script will run all tests below for a specific typeshed directory. If a @@ -46,7 +46,7 @@ be selected. A summary of the results will be printed to the terminal. You must provide a single argument which is a path to the stubs to test, like so: `stdlib/os` or `stubs/requests`. -Run `python scripts/runtests.py --help` for information on the various configuration options +Run `python tests/runtests.py --help` for information on the various configuration options for this script. Note that if you use the `--run-stubtest` flag with the stdlib stubs, whether or not the test passes will depend on the exact version of Python you're using, as well as various other details regarding your local environment. diff --git a/tests/check_typeshed_structure.py b/tests/check_typeshed_structure.py index cc6180fe322a..20b329878199 100755 --- a/tests/check_typeshed_structure.py +++ b/tests/check_typeshed_structure.py @@ -15,12 +15,15 @@ from parse_metadata import read_metadata from utils import ( REQS_FILE, + TEST_CASES_DIR, + TESTS_DIR, VERSIONS_RE, get_all_testcase_directories, get_gitignore_spec, parse_requirements, spec_matches_path, strip_comments, + tests_path, ) extension_descriptions = {".pyi": "stub", ".py": ".py"} @@ -73,13 +76,15 @@ def check_stubs() -> None: ), f"Directory name must be a valid distribution name: {dist}" assert not dist.name.startswith("types-"), f"Directory name not allowed to start with 'types-': {dist}" - allowed = {"METADATA.toml", "README", "README.md", "README.rst", "@tests"} + allowed = {"METADATA.toml", "README", "README.md", "README.rst", TESTS_DIR} assert_consistent_filetypes(dist, kind=".pyi", allowed=allowed) - tests_dir = dist / "@tests" + tests_dir = tests_path(dist.name) if tests_dir.exists() and tests_dir.is_dir(): py_files_present = any(file.suffix == ".py" for file in tests_dir.iterdir()) - error_message = "Test-case files must be in an `@tests/test_cases/` directory, not in the `@tests/` directory" + error_message = ( + f"Test-case files must be in an `{TESTS_DIR}/{TEST_CASES_DIR}` directory, not in the `{TESTS_DIR}` directory" + ) assert not py_files_present, error_message @@ -101,7 +106,7 @@ def check_test_cases() -> None: """Check that the test_cases directory contains only the correct files.""" for _, testcase_dir in get_all_testcase_directories(): assert_consistent_filetypes(testcase_dir, kind=".py", allowed={"README.md"}, allow_nonidentifier_filenames=True) - bad_test_case_filename = 'Files in a `test_cases` directory must have names starting with "check_"; got "{}"' + bad_test_case_filename = f'Files in a `{TEST_CASES_DIR}` directory must have names starting with "check_"; got "{{}}"' for file in testcase_dir.rglob("*.py"): assert file.stem.startswith("check_"), bad_test_case_filename.format(file) diff --git a/tests/regr_test.py b/tests/regr_test.py index db6d8d08bfed..d4aefc923370 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -24,18 +24,18 @@ from parse_metadata import get_recursive_requirements, read_metadata from utils import ( PYTHON_VERSION, - PackageInfo, + TEST_CASES_DIR, + DistributionTests, colored, + distribution_info, get_all_testcase_directories, get_mypy_req, print_error, - testcase_dir_from_package_name, venv_python, ) ReturnCode: TypeAlias = int -TEST_CASES = "test_cases" VENV_DIR = ".venv" TYPESHED = "typeshed" @@ -43,17 +43,13 @@ SUPPORTED_VERSIONS = ["3.12", "3.11", "3.10", "3.9", "3.8"] -def package_with_test_cases(package_name: str) -> PackageInfo: - """Helper function for argument-parsing""" +def distribution_with_test_cases(distribution_name: str) -> DistributionTests: + """Helper function for argument-parsing.""" - if package_name == "stdlib": - return PackageInfo("stdlib", Path(TEST_CASES)) - test_case_dir = testcase_dir_from_package_name(package_name) - if test_case_dir.is_dir(): - if not os.listdir(test_case_dir): - raise argparse.ArgumentTypeError(f"{package_name!r} has a 'test_cases' directory but it is empty!") - return PackageInfo(package_name, test_case_dir) - raise argparse.ArgumentTypeError(f"No test cases found for {package_name!r}!") + try: + return distribution_info(distribution_name) + except RuntimeError as exc: + raise argparse.ArgumentTypeError(str(exc)) from exc class Verbosity(IntEnum): @@ -65,7 +61,7 @@ class Verbosity(IntEnum): parser = argparse.ArgumentParser(description="Script to run mypy against various test cases for typeshed's stubs") parser.add_argument( "packages_to_test", - type=package_with_test_cases, + type=distribution_with_test_cases, nargs="*", action="extend", help=( @@ -118,13 +114,13 @@ def verbose_log(msg: str) -> None: _PRINT_QUEUE.put(colored(msg, "blue")) -def setup_testcase_dir(package: PackageInfo, tempdir: Path, verbosity: Verbosity) -> None: +def setup_testcase_dir(package: DistributionTests, tempdir: Path, verbosity: Verbosity) -> None: if verbosity is verbosity.VERBOSE: verbose_log(f"{package.name}: Setting up testcase dir in {tempdir}") # --warn-unused-ignores doesn't work for files inside typeshed. # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory, # and run the test cases inside of that. - shutil.copytree(package.test_case_directory, tempdir / TEST_CASES) + shutil.copytree(package.test_cases_path, tempdir / TEST_CASES_DIR) if package.is_stdlib: return @@ -163,10 +159,10 @@ def setup_testcase_dir(package: PackageInfo, tempdir: Path, verbosity: Verbosity def run_testcases( - package: PackageInfo, version: str, platform: str, *, tempdir: Path, verbosity: Verbosity + package: DistributionTests, version: str, platform: str, *, tempdir: Path, verbosity: Verbosity ) -> subprocess.CompletedProcess[str]: env_vars = dict(os.environ) - new_test_case_dir = tempdir / TEST_CASES + new_test_case_dir = tempdir / TEST_CASES_DIR # "--enable-error-code ignore-without-code" is purposefully omitted. # See https://github.com/python/typeshed/pull/8083 @@ -239,14 +235,16 @@ def print_description(self, *, verbosity: Verbosity) -> None: if self.code: print(f"{self.command_run}:", end=" ") print_error("FAILURE\n") - replacements = (str(self.tempdir / TEST_CASES), str(self.test_case_dir)) + replacements = (str(self.tempdir / TEST_CASES_DIR), str(self.test_case_dir)) if self.stderr: print_error(self.stderr, fix_path=replacements) if self.stdout: print_error(self.stdout, fix_path=replacements) -def test_testcase_directory(package: PackageInfo, version: str, platform: str, *, verbosity: Verbosity, tempdir: Path) -> Result: +def test_testcase_directory( + package: DistributionTests, version: str, platform: str, *, verbosity: Verbosity, tempdir: Path +) -> Result: msg = f"mypy --platform {platform} --python-version {version} on the " msg += "standard library test cases" if package.is_stdlib else f"test cases for {package.name!r}" if verbosity > Verbosity.QUIET: @@ -258,7 +256,7 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, * command_run=msg, stderr=proc_info.stderr, stdout=proc_info.stdout, - test_case_dir=package.test_case_directory, + test_case_dir=package.test_cases_path, tempdir=tempdir, ) @@ -278,13 +276,13 @@ def print_queued_messages(ev: threading.Event) -> None: def concurrently_run_testcases( stack: ExitStack, - testcase_directories: list[PackageInfo], + testcase_directories: list[DistributionTests], verbosity: Verbosity, platforms_to_test: list[str], versions_to_test: list[str], ) -> list[Result]: packageinfo_to_tempdir = { - package_info: Path(stack.enter_context(tempfile.TemporaryDirectory())) for package_info in testcase_directories + distribution_info: Path(stack.enter_context(tempfile.TemporaryDirectory())) for distribution_info in testcase_directories } to_do: list[Callable[[], Result]] = [] for testcase_dir, tempdir in packageinfo_to_tempdir.items(): diff --git a/scripts/runtests.py b/tests/runtests.py similarity index 96% rename from scripts/runtests.py rename to tests/runtests.py index 7a17c299f01f..2a4f445af12e 100755 --- a/scripts/runtests.py +++ b/tests/runtests.py @@ -10,6 +10,8 @@ from pathlib import Path from typing import Any +from utils import TEST_CASES_DIR, test_cases_path + try: from termcolor import colored # pyright: ignore[reportAssignmentType] except ImportError: @@ -20,7 +22,6 @@ def colored(text: str, color: str | None = None, **kwargs: Any) -> str: # type: _STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json" _TESTCASES_CONFIG_FILE = "pyrightconfig.testcases.json" -_TESTCASES = "test_cases" _NPX_ERROR_PATTERN = r"error (runn|find)ing npx" _NPX_ERROR_MESSAGE = colored("\nSkipping Pyright tests: npx is not installed or can't be run!", "yellow") _SUCCESS = colored("Success", "green") @@ -132,10 +133,10 @@ def main() -> None: print("\nRunning pytype...") pytype_result = subprocess.run([sys.executable, "tests/pytype_test.py", path]) - test_cases_path = Path(path) / "@tests" / _TESTCASES if folder == "stubs" else Path(_TESTCASES) - if not test_cases_path.exists(): + cases_path = test_cases_path(stub if folder == "stubs" else "stdlib") + if not cases_path.exists(): # No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests. - print(colored(f"\nRegression tests: No {_TESTCASES} folder for {stub!r}!", "green")) + print(colored(f"\nRegression tests: No {TEST_CASES_DIR} folder for {stub!r}!", "green")) pyright_testcases_returncode = 0 pyright_testcases_skipped = False regr_test_returncode = 0 @@ -144,7 +145,7 @@ def main() -> None: command = [ sys.executable, "tests/pyright_test.py", - str(test_cases_path), + str(cases_path), "--pythonversion", python_version, "-p", diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index a9a123fa8ec7..513eeab67de0 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -13,7 +13,7 @@ from typing import NoReturn from parse_metadata import NoSuchStubError, get_recursive_requirements, read_metadata -from utils import PYTHON_VERSION, colored, get_mypy_req, print_divider, print_error, print_success_msg +from utils import PYTHON_VERSION, colored, get_mypy_req, print_divider, print_error, print_success_msg, tests_path def run_stubtest( @@ -112,10 +112,10 @@ def run_stubtest( # "bisect" to see which variables are actually needed. stubtest_env = os.environ | {"MYPYPATH": mypypath, "MYPY_FORCE_COLOR": "1"} - allowlist_path = dist / "@tests/stubtest_allowlist.txt" + allowlist_path = tests_path(dist_name) / "stubtest_allowlist.txt" if allowlist_path.exists(): stubtest_cmd.extend(["--allowlist", str(allowlist_path)]) - platform_allowlist = dist / f"@tests/stubtest_allowlist_{sys.platform}.txt" + platform_allowlist = tests_path(dist_name) / f"stubtest_allowlist_{sys.platform}.txt" if platform_allowlist.exists(): stubtest_cmd.extend(["--allowlist", str(platform_allowlist)]) @@ -271,7 +271,7 @@ def setup_uwsgi_stubtest_command(dist: Path, venv_dir: Path, stubtest_cmd: list[ so both scripts will be cleaned up after this function has been executed. """ - uwsgi_ini = dist / "@tests/uwsgi.ini" + uwsgi_ini = tests_path(dist.name) / "uwsgi.ini" if sys.platform == "win32": print_error("uWSGI is not supported on Windows") diff --git a/tests/utils.py b/tests/utils.py index 345194ae78f9..6fd21d34c113 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os import re import sys from collections.abc import Iterable, Mapping @@ -23,6 +22,8 @@ def colored(text: str, color: str | None = None, **kwargs: Any) -> str: # type: PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}" +STUBS_PATH = Path("stubs") + # A backport of functools.cache for Python <3.9 # This module is imported by mypy_test.py, which needs to run on 3.8 in CI @@ -109,30 +110,55 @@ def get_mypy_req() -> str: # ==================================================================== -# Getting test-case directories from package names +# Test Directories # ==================================================================== -class PackageInfo(NamedTuple): +TESTS_DIR: Final = "@tests" +TEST_CASES_DIR: Final = "test_cases" + + +class DistributionTests(NamedTuple): name: str - test_case_directory: Path + test_cases_path: Path @property def is_stdlib(self) -> bool: return self.name == "stdlib" -def testcase_dir_from_package_name(package_name: str) -> Path: - return Path("stubs", package_name, "@tests/test_cases") +def distribution_info(distribution_name: str) -> DistributionTests: + if distribution_name == "stdlib": + return DistributionTests("stdlib", test_cases_path("stdlib")) + test_path = test_cases_path(distribution_name) + if test_path.is_dir(): + if not list(test_path.iterdir()): + raise RuntimeError(f"{distribution_name!r} has a '{TEST_CASES_DIR}' directory but it is empty!") + return DistributionTests(distribution_name, test_path) + raise RuntimeError(f"No test cases found for {distribution_name!r}!") + + +def tests_path(distribution_name: str) -> Path: + assert distribution_name != "stdlib" + return STUBS_PATH / distribution_name / TESTS_DIR + + +def test_cases_path(distribution_name: str) -> Path: + if distribution_name == "stdlib": + return Path(TEST_CASES_DIR) + else: + return tests_path(distribution_name) / TEST_CASES_DIR -def get_all_testcase_directories() -> list[PackageInfo]: - testcase_directories: list[PackageInfo] = [] - for package_name in os.listdir("stubs"): - potential_testcase_dir = testcase_dir_from_package_name(package_name) - if potential_testcase_dir.is_dir(): - testcase_directories.append(PackageInfo(package_name, potential_testcase_dir)) - return [PackageInfo("stdlib", Path("test_cases")), *sorted(testcase_directories)] +def get_all_testcase_directories() -> list[DistributionTests]: + testcase_directories: list[DistributionTests] = [] + for distribution_path in STUBS_PATH.iterdir(): + try: + pkg_info = distribution_info(distribution_path.name) + except RuntimeError: + continue + testcase_directories.append(pkg_info) + return [distribution_info("stdlib"), *sorted(testcase_directories)] # ====================================================================