From 56e755b2954ee4b245e2245932e028c49d921200 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Sun, 25 Dec 2022 01:05:28 +0000 Subject: [PATCH 01/17] Refactor to use subprocess; other small improvements --- tests/mypy_test.py | 53 +++++++++++++++++++--------------------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 55f846b3d8f1..7effe2b39b76 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -6,11 +6,10 @@ import argparse import os import re +import subprocess import sys import tempfile -from contextlib import redirect_stderr, redirect_stdout from dataclasses import dataclass -from io import StringIO from itertools import product from pathlib import Path from typing import TYPE_CHECKING, Any, NamedTuple @@ -32,8 +31,9 @@ strip_comments, ) +# Fail early if mypy isn't installed try: - from mypy.api import run as mypy_run + import mypy # noqa: F401 except ImportError: print_error("Cannot import mypy. Did you install it?") sys.exit(1) @@ -108,7 +108,7 @@ class TestConfig: def log(args: TestConfig, *varargs: object) -> None: if args.verbose >= 2: - print(*varargs) + print(colored(" ".join(map(str, varargs)), "blue")) def match(path: Path, args: TestConfig) -> bool: @@ -204,7 +204,12 @@ def add_configuration(configurations: list[MypyDistConf], distribution: str) -> configurations.append(MypyDistConf(module_name, values.copy())) -def run_mypy(args: TestConfig, configurations: list[MypyDistConf], files: list[Path], *, testing_stdlib: bool) -> ReturnCode: +def run_mypy( + args: TestConfig, configurations: list[MypyDistConf], files: list[Path], *, testing_stdlib: bool, mypypath: str | None = None +) -> ReturnCode: + env_vars = dict(os.environ) + if mypypath is not None: + env_vars["MYPYPATH"] = mypypath with tempfile.NamedTemporaryFile("w+") as temp: temp.write("[mypy]\n") for dist_conf in configurations: @@ -217,25 +222,17 @@ def run_mypy(args: TestConfig, configurations: list[MypyDistConf], files: list[P mypy_args = [*flags, *map(str, files)] if args.verbose: print("running mypy", " ".join(mypy_args)) - stdout_redirect, stderr_redirect = StringIO(), StringIO() - with redirect_stdout(stdout_redirect), redirect_stderr(stderr_redirect): - returned_stdout, returned_stderr, exit_code = mypy_run(mypy_args) - - if exit_code: + mypy_command = [sys.executable, "-m", "mypy"] + mypy_args + result = subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars) + if result.returncode: print_error("failure\n") - captured_stdout = stdout_redirect.getvalue() - captured_stderr = stderr_redirect.getvalue() - if returned_stderr: - print_error(returned_stderr) - if captured_stderr: - print_error(captured_stderr) - if returned_stdout: - print_error(returned_stdout) - if captured_stdout: - print_error(captured_stdout, end="") + if result.stdout: + print_error(result.stdout) + if result.stderr: + print_error(result.stderr) else: print_success_msg() - return exit_code + return result.returncode def get_mypy_flags(args: TestConfig, temp_name: str, *, testing_stdlib: bool) -> list[str]: @@ -313,20 +310,14 @@ def test_third_party_distribution(distribution: str, args: TestConfig) -> TestRe if not files and args.filter: return TestResults(0, 0) - print(f"testing {distribution} ({len(files)} files)... ", end="") + print(f"testing {distribution} ({len(files)} files)... ", end="", flush=True) if not files: print_error("no files found") sys.exit(1) - prev_mypypath = os.getenv("MYPYPATH") - os.environ["MYPYPATH"] = os.pathsep.join(str(Path("stubs", dist)) for dist in seen_dists) - code = run_mypy(args, configurations, files, testing_stdlib=False) - if prev_mypypath is None: - del os.environ["MYPYPATH"] - else: - os.environ["MYPYPATH"] = prev_mypypath - + mypypath = os.pathsep.join(str(Path("stubs", dist)) for dist in seen_dists) + code = run_mypy(args, configurations, files, mypypath=mypypath, testing_stdlib=False) return TestResults(code, len(files)) @@ -417,5 +408,5 @@ def main() -> None: try: main() except KeyboardInterrupt: - print_error("\n\n!!!\nTest aborted due to KeyboardInterrupt\n!!!") + print_error("\n\nTest aborted due to KeyboardInterrupt!") sys.exit(1) From ee036e70ef4f5ff1dee25eb03739a0aca5414409 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Sun, 25 Dec 2022 01:05:44 +0000 Subject: [PATCH 02/17] Allow non-types dependencies --- tests/mypy_test.py | 183 +++++++++++++++++++++++++++++++++------------ tests/utils.py | 2 +- 2 files changed, 136 insertions(+), 49 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 7effe2b39b76..61211e27d05a 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import concurrent.futures import os import re import subprocess @@ -12,6 +13,7 @@ from dataclasses import dataclass from itertools import product from pathlib import Path +from threading import Lock from typing import TYPE_CHECKING, Any, NamedTuple if TYPE_CHECKING: @@ -22,9 +24,13 @@ import tomli from utils import ( VERSIONS_RE as VERSION_LINE_RE, + PackageDependencies, + VenvInfo, colored, get_gitignore_spec, + get_mypy_req, get_recursive_requirements, + make_venv, print_error, print_success_msg, spec_matches_path, @@ -205,7 +211,14 @@ def add_configuration(configurations: list[MypyDistConf], distribution: str) -> def run_mypy( - args: TestConfig, configurations: list[MypyDistConf], files: list[Path], *, testing_stdlib: bool, mypypath: str | None = None + args: TestConfig, + configurations: list[MypyDistConf], + files: list[Path], + *, + testing_stdlib: bool, + non_types_dependencies: bool, + python_exe: str, + mypypath: str | None = None, ) -> ReturnCode: env_vars = dict(os.environ) if mypypath is not None: @@ -218,11 +231,36 @@ def run_mypy( temp.write(f"{k} = {v}\n") temp.flush() - flags = get_mypy_flags(args, temp.name, testing_stdlib=testing_stdlib) + flags = [ + "--python-version", + args.version, + "--show-traceback", + "--warn-incomplete-stub", + "--show-error-codes", + "--no-error-summary", + "--platform", + args.platform, + "--custom-typeshed-dir", + str(Path(__file__).parent.parent), + "--strict", + # Stub completion is checked by pyright (--allow-*-defs) + "--allow-untyped-defs", + "--allow-incomplete-defs", + "--allow-subclassing-any", # Needed until we can use non-types dependencies #5768 + "--enable-error-code", + "ignore-without-code", + "--config-file", + temp.name, + ] + if not testing_stdlib: + flags.append("--explicit-package-bases") + if not non_types_dependencies: + flags.append("--no-site-packages") + mypy_args = [*flags, *map(str, files)] + mypy_command = [python_exe, "-m", "mypy"] + mypy_args if args.verbose: - print("running mypy", " ".join(mypy_args)) - mypy_command = [sys.executable, "-m", "mypy"] + mypy_args + print(colored(f"running {' '.join(mypy_command)}", "blue")) result = subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars) if result.returncode: print_error("failure\n") @@ -235,34 +273,6 @@ def run_mypy( return result.returncode -def get_mypy_flags(args: TestConfig, temp_name: str, *, testing_stdlib: bool) -> list[str]: - flags = [ - "--python-version", - args.version, - "--show-traceback", - "--warn-incomplete-stub", - "--show-error-codes", - "--no-error-summary", - "--platform", - args.platform, - "--no-site-packages", - "--custom-typeshed-dir", - str(Path(__file__).parent.parent), - "--strict", - # Stub completion is checked by pyright (--allow-*-defs) - "--allow-untyped-defs", - "--allow-incomplete-defs", - "--allow-subclassing-any", # Needed until we can use non-types dependencies #5768 - "--enable-error-code", - "ignore-without-code", - "--config-file", - temp_name, - ] - if not testing_stdlib: - flags.append("--explicit-package-bases") - return flags - - def add_third_party_files( distribution: str, files: list[Path], args: TestConfig, configurations: list[MypyDistConf], seen_dists: set[str] ) -> None: @@ -295,7 +305,9 @@ class TestResults(NamedTuple): files_checked: int -def test_third_party_distribution(distribution: str, args: TestConfig) -> TestResults: +def test_third_party_distribution( + distribution: str, args: TestConfig, python_exe: str, *, non_types_dependencies: bool +) -> TestResults: """Test the stubs of a third-party distribution. Return a tuple, where the first element indicates mypy's return code @@ -317,7 +329,17 @@ def test_third_party_distribution(distribution: str, args: TestConfig) -> TestRe sys.exit(1) mypypath = os.pathsep.join(str(Path("stubs", dist)) for dist in seen_dists) - code = run_mypy(args, configurations, files, mypypath=mypypath, testing_stdlib=False) + if args.verbose: + print(colored(f"\n{mypypath=}", "blue")) + code = run_mypy( + args, + configurations, + files, + python_exe=python_exe, + mypypath=mypypath, + testing_stdlib=False, + non_types_dependencies=non_types_dependencies, + ) return TestResults(code, len(files)) @@ -334,19 +356,72 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults: add_files(files, (stdlib / name), args) if files: - print(f"Testing stdlib ({len(files)} files)...") - print("Running mypy " + " ".join(get_mypy_flags(args, "/tmp/...", testing_stdlib=True))) - this_code = run_mypy(args, [], files, testing_stdlib=True) + print(f"Testing stdlib ({len(files)} files)...", end="", flush=True) + this_code = run_mypy(args, [], files, python_exe=sys.executable, testing_stdlib=True, non_types_dependencies=False) code = max(code, this_code) return TestResults(code, len(files)) -def test_third_party_stubs(code: int, args: TestConfig) -> TestResults: +_PRINT_LOCK = Lock() +_PYTHON_EXE_MAPPING: dict[str, VenvInfo] = {} + + +def setup_venv_for_distribution(distribution: str, tempdir: Path) -> tuple[str, VenvInfo]: + venv_dir = tempdir / f".venv-{distribution}" + return distribution, make_venv(venv_dir) + + +def install_requirements_for_distribution( + distribution: str, pip_exe: str, args: TestConfig, external_requirements: tuple[str, ...] +) -> None: + # Use --no-cache-dir to avoid issues with concurrent read/writes to the cache + pip_command = [pip_exe, "install", get_mypy_req(), *external_requirements, "--no-cache-dir"] + if args.verbose: + with _PRINT_LOCK: + print(colored(f"pip installing the following requirements for {distribution!r}: {external_requirements}", "blue")) + try: + subprocess.run(pip_command, check=True, capture_output=True, text=True) + except subprocess.CalledProcessError as e: + print(e.stderr) + raise + + +def setup_virtual_environments(distributions: dict[str, PackageDependencies], args: TestConfig, tempdir: Path) -> None: + distributions_needing_venvs: dict[str, PackageDependencies] = {} + for distribution, requirements in distributions.items(): + if requirements.external_pkgs: + distributions_needing_venvs[distribution] = requirements + else: + _PYTHON_EXE_MAPPING[distribution] = VenvInfo(pip_exe="", python_exe=sys.executable) + + if args.verbose: + print(colored(f"Setting up venvs for {list(distributions_needing_venvs)}...", "blue")) + + with concurrent.futures.ThreadPoolExecutor() as executor: + venv_info_futures = [ + executor.submit(setup_venv_for_distribution, distribution, tempdir) for distribution in distributions_needing_venvs + ] + for venv_info_future in concurrent.futures.as_completed(venv_info_futures): + distribution, venv_info = venv_info_future.result() + _PYTHON_EXE_MAPPING[distribution] = venv_info + + # Limit workers to 5 at a time, since this makes network requests + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + futures = [] + for distribution, requirements in distributions_needing_venvs.items(): + pip_exe = _PYTHON_EXE_MAPPING[distribution].pip_exe + futures.append( + executor.submit(install_requirements_for_distribution, distribution, pip_exe, args, requirements.external_pkgs) + ) + concurrent.futures.wait(futures) + + +def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestResults: print("Testing third-party packages...") - print("Running mypy " + " ".join(get_mypy_flags(args, "/tmp/...", testing_stdlib=False))) files_checked = 0 gitignore_spec = get_gitignore_spec() + distributions_to_check: dict[str, PackageDependencies] = {} for distribution in sorted(os.listdir("stubs")): distribution_path = Path("stubs", distribution) @@ -359,14 +434,24 @@ def test_third_party_stubs(code: int, args: TestConfig) -> TestResults: or Path("stubs") in args.filter or any(distribution_path in path.parents for path in args.filter) ): - this_code, checked = test_third_party_distribution(distribution, args) - code = max(code, this_code) - files_checked += checked + distributions_to_check[distribution] = get_recursive_requirements(distribution) + + if not _PYTHON_EXE_MAPPING: + setup_virtual_environments(distributions_to_check, args, tempdir) + + for distribution, requirements in distributions_to_check.items(): + has_non_types_dependencies = bool(requirements.external_pkgs) + python_to_use = _PYTHON_EXE_MAPPING[distribution].python_exe + this_code, checked = test_third_party_distribution( + distribution, args, python_exe=python_to_use, non_types_dependencies=has_non_types_dependencies + ) + code = max(code, this_code) + files_checked += checked return TestResults(code, files_checked) -def test_typeshed(code: int, args: TestConfig) -> TestResults: +def test_typeshed(code: int, args: TestConfig, tempdir: Path) -> TestResults: print(f"*** Testing Python {args.version} on {args.platform}") files_checked_this_version = 0 stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs") @@ -376,7 +461,7 @@ def test_typeshed(code: int, args: TestConfig) -> TestResults: print() if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter): - code, third_party_files_checked = test_third_party_stubs(code, args) + code, third_party_files_checked = test_third_party_stubs(code, args, tempdir) files_checked_this_version += third_party_files_checked print() @@ -391,10 +476,12 @@ def main() -> None: exclude = args.exclude or [] code = 0 total_files_checked = 0 - for version, platform in product(versions, platforms): - config = TestConfig(args.verbose, filter, exclude, version, platform) - code, files_checked_this_version = test_typeshed(code, args=config) - total_files_checked += files_checked_this_version + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) + for version, platform in product(versions, platforms): + config = TestConfig(args.verbose, filter, exclude, version, platform) + code, files_checked_this_version = test_typeshed(code, args=config, tempdir=td_path) + total_files_checked += files_checked_this_version if code: print_error(f"--- exit status {code}, {total_files_checked} files checked ---") sys.exit(code) diff --git a/tests/utils.py b/tests/utils.py index cdfba98000b9..98cc829db296 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -142,7 +142,7 @@ def make_venv(venv_dir: Path) -> VenvInfo: try: venv.create(venv_dir, with_pip=True, clear=True) except subprocess.CalledProcessError as e: - if "ensurepip" in e.cmd: + if "ensurepip" in e.cmd and "KeyboardInterrupt" not in e.stdout: print_error( "stubtest requires a Python installation with ensurepip. " "If on Linux, you may need to install the python3-venv package." From ca484c4e39bfd56c9d55eb09b91151f45e7a3011 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Dec 2022 16:50:57 +0000 Subject: [PATCH 03/17] Share venvs between packages with common requirements --- tests/mypy_test.py | 88 +++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 32 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 61211e27d05a..8172053f3ae8 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -10,6 +10,8 @@ import subprocess import sys import tempfile +import time +from collections import defaultdict from dataclasses import dataclass from itertools import product from pathlib import Path @@ -246,7 +248,7 @@ def run_mypy( # Stub completion is checked by pyright (--allow-*-defs) "--allow-untyped-defs", "--allow-incomplete-defs", - "--allow-subclassing-any", # Needed until we can use non-types dependencies #5768 + "--allow-subclassing-any", # TODO: Do we still need this now that non-types dependencies are allowed? "--enable-error-code", "ignore-without-code", "--config-file", @@ -364,22 +366,21 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults: _PRINT_LOCK = Lock() -_PYTHON_EXE_MAPPING: dict[str, VenvInfo] = {} +_DISTRIBUTION_TO_VENV_MAPPING: dict[str, VenvInfo] = {} -def setup_venv_for_distribution(distribution: str, tempdir: Path) -> tuple[str, VenvInfo]: - venv_dir = tempdir / f".venv-{distribution}" - return distribution, make_venv(venv_dir) +def setup_venv_for_external_requirements_set(requirements_set: frozenset[str], tempdir: Path) -> tuple[frozenset[str], VenvInfo]: + reqs_joined = "-".join(sorted(requirements_set)) + venv_dir = tempdir / f".venv-{reqs_joined}" + return requirements_set, make_venv(venv_dir) -def install_requirements_for_distribution( - distribution: str, pip_exe: str, args: TestConfig, external_requirements: tuple[str, ...] -) -> None: +def install_requirements_for_venv(venv_info: VenvInfo, args: TestConfig, external_requirements: frozenset[str]) -> None: # Use --no-cache-dir to avoid issues with concurrent read/writes to the cache - pip_command = [pip_exe, "install", get_mypy_req(), *external_requirements, "--no-cache-dir"] + pip_command = [venv_info.pip_exe, "install", get_mypy_req(), *sorted(external_requirements), "--no-cache-dir"] if args.verbose: with _PRINT_LOCK: - print(colored(f"pip installing the following requirements for {distribution!r}: {external_requirements}", "blue")) + print(colored(f"Running {pip_command}", "blue")) try: subprocess.run(pip_command, check=True, capture_output=True, text=True) except subprocess.CalledProcessError as e: @@ -388,33 +389,55 @@ def install_requirements_for_distribution( def setup_virtual_environments(distributions: dict[str, PackageDependencies], args: TestConfig, tempdir: Path) -> None: - distributions_needing_venvs: dict[str, PackageDependencies] = {} - for distribution, requirements in distributions.items(): + no_external_dependencies_venv = VenvInfo(pip_exe="", python_exe=sys.executable) + external_requirements_to_distributions: defaultdict[frozenset[str], list[str]] = defaultdict(list) + num_pkgs_with_external_reqs = 0 + + for distribution_name, requirements in distributions.items(): if requirements.external_pkgs: - distributions_needing_venvs[distribution] = requirements + num_pkgs_with_external_reqs += 1 + external_requirements_to_distributions[frozenset(requirements.external_pkgs)].append(distribution_name) else: - _PYTHON_EXE_MAPPING[distribution] = VenvInfo(pip_exe="", python_exe=sys.executable) + _DISTRIBUTION_TO_VENV_MAPPING[distribution_name] = no_external_dependencies_venv + + requirements_sets_to_venvs: dict[frozenset[str], VenvInfo] = {} if args.verbose: - print(colored(f"Setting up venvs for {list(distributions_needing_venvs)}...", "blue")) + num_venvs = len(external_requirements_to_distributions) + msg = f"Setting up {num_venvs} venvs for {num_pkgs_with_external_reqs} distributions... " + print(colored(msg, "blue"), end="", flush=True) + venv_start_time = time.perf_counter() with concurrent.futures.ThreadPoolExecutor() as executor: venv_info_futures = [ - executor.submit(setup_venv_for_distribution, distribution, tempdir) for distribution in distributions_needing_venvs + executor.submit(setup_venv_for_external_requirements_set, requirements_set, tempdir) + for requirements_set in external_requirements_to_distributions ] for venv_info_future in concurrent.futures.as_completed(venv_info_futures): - distribution, venv_info = venv_info_future.result() - _PYTHON_EXE_MAPPING[distribution] = venv_info + requirements_set, venv_info = venv_info_future.result() + requirements_sets_to_venvs[requirements_set] = venv_info - # Limit workers to 5 at a time, since this makes network requests - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - futures = [] - for distribution, requirements in distributions_needing_venvs.items(): - pip_exe = _PYTHON_EXE_MAPPING[distribution].pip_exe - futures.append( - executor.submit(install_requirements_for_distribution, distribution, pip_exe, args, requirements.external_pkgs) - ) - concurrent.futures.wait(futures) + if args.verbose: + venv_elapsed_time = time.perf_counter() - venv_start_time + print(colored(f"took {venv_elapsed_time:.2f} seconds", "blue")) + pip_start_time = time.perf_counter() + + # Limit workers to 10 at a time, since this makes network requests + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + pip_install_futures = [ + executor.submit(install_requirements_for_venv, venv_info, args, requirements_set) + for requirements_set, venv_info in requirements_sets_to_venvs.items() + ] + concurrent.futures.wait(pip_install_futures) + + if args.verbose: + pip_elapsed_time = time.perf_counter() - pip_start_time + msg = f"Combined time for installing requirements across all venvs: {pip_elapsed_time:.2f} seconds" + print(colored(msg, "blue")) + + for requirements_set, distribution_list in external_requirements_to_distributions.items(): + venv_to_use = requirements_sets_to_venvs[requirements_set] + _DISTRIBUTION_TO_VENV_MAPPING.update(dict.fromkeys(distribution_list, venv_to_use)) def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestResults: @@ -436,14 +459,15 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe ): distributions_to_check[distribution] = get_recursive_requirements(distribution) - if not _PYTHON_EXE_MAPPING: + if not _DISTRIBUTION_TO_VENV_MAPPING: setup_virtual_environments(distributions_to_check, args, tempdir) - for distribution, requirements in distributions_to_check.items(): - has_non_types_dependencies = bool(requirements.external_pkgs) - python_to_use = _PYTHON_EXE_MAPPING[distribution].python_exe + assert len(_DISTRIBUTION_TO_VENV_MAPPING) == len(distributions_to_check) + + for distribution, venv_info in _DISTRIBUTION_TO_VENV_MAPPING.items(): + venv_python = venv_info.python_exe this_code, checked = test_third_party_distribution( - distribution, args, python_exe=python_to_use, non_types_dependencies=has_non_types_dependencies + distribution, args, python_exe=venv_python, non_types_dependencies=(venv_python != sys.executable) ) code = max(code, this_code) files_checked += checked From db260f2cea479e8d7e7340868eedeaea09312e85 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Dec 2022 16:55:15 +0000 Subject: [PATCH 04/17] Here's what happens if we add loads of non-types requirements --- .github/workflows/tests.yml | 2 +- stubs/Pillow/METADATA.toml | 1 + stubs/SQLAlchemy/METADATA.toml | 1 + stubs/paramiko/METADATA.toml | 2 +- stubs/requests/METADATA.toml | 2 +- 5 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e5285416405c..7f6b934882e2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,7 +87,7 @@ jobs: cache: pip cache-dependency-path: requirements-tests.txt - run: pip install -r requirements-tests.txt - - run: python ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }} + - run: python ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }} --verbose regression-tests: name: Run mypy on the test cases diff --git a/stubs/Pillow/METADATA.toml b/stubs/Pillow/METADATA.toml index 9f95a60c30bd..9776ddcef55d 100644 --- a/stubs/Pillow/METADATA.toml +++ b/stubs/Pillow/METADATA.toml @@ -1 +1,2 @@ +requires = ["packaging"] version = "9.3.*" diff --git a/stubs/SQLAlchemy/METADATA.toml b/stubs/SQLAlchemy/METADATA.toml index 19ac9f269036..65a458af512f 100644 --- a/stubs/SQLAlchemy/METADATA.toml +++ b/stubs/SQLAlchemy/METADATA.toml @@ -1,3 +1,4 @@ +requires = ["tomli"] version = "1.4.45" extra_description = """\ The `sqlalchemy-stubs` package is an alternative to this package and also \ diff --git a/stubs/paramiko/METADATA.toml b/stubs/paramiko/METADATA.toml index 1c52e4c68492..fc1fee0cd8cd 100644 --- a/stubs/paramiko/METADATA.toml +++ b/stubs/paramiko/METADATA.toml @@ -1,5 +1,5 @@ version = "2.12.*" -requires = ["types-cryptography"] +requires = ["cryptography"] [tool.stubtest] # linux and darwin are equivalent diff --git a/stubs/requests/METADATA.toml b/stubs/requests/METADATA.toml index 0fe1db2749cc..571aac6a8f99 100644 --- a/stubs/requests/METADATA.toml +++ b/stubs/requests/METADATA.toml @@ -1,5 +1,5 @@ version = "2.28.*" -requires = ["types-urllib3<1.27"] # keep in sync with requests's setup.py +requires = ["types-urllib3<1.27", "numpy"] # keep in sync with requests's setup.py [tool.stubtest] ignore_missing_stub = false From 28973be65ab05d0e2a3670c930fc9e8b479d8884 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Dec 2022 16:59:47 +0000 Subject: [PATCH 05/17] Revert "Here's what happens if we add loads of non-types requirements" This reverts commit db260f2cea479e8d7e7340868eedeaea09312e85. --- .github/workflows/tests.yml | 2 +- stubs/Pillow/METADATA.toml | 1 - stubs/SQLAlchemy/METADATA.toml | 1 - stubs/paramiko/METADATA.toml | 2 +- stubs/requests/METADATA.toml | 2 +- 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7f6b934882e2..e5285416405c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,7 +87,7 @@ jobs: cache: pip cache-dependency-path: requirements-tests.txt - run: pip install -r requirements-tests.txt - - run: python ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }} --verbose + - run: python ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }} regression-tests: name: Run mypy on the test cases diff --git a/stubs/Pillow/METADATA.toml b/stubs/Pillow/METADATA.toml index 9776ddcef55d..9f95a60c30bd 100644 --- a/stubs/Pillow/METADATA.toml +++ b/stubs/Pillow/METADATA.toml @@ -1,2 +1 @@ -requires = ["packaging"] version = "9.3.*" diff --git a/stubs/SQLAlchemy/METADATA.toml b/stubs/SQLAlchemy/METADATA.toml index 65a458af512f..19ac9f269036 100644 --- a/stubs/SQLAlchemy/METADATA.toml +++ b/stubs/SQLAlchemy/METADATA.toml @@ -1,4 +1,3 @@ -requires = ["tomli"] version = "1.4.45" extra_description = """\ The `sqlalchemy-stubs` package is an alternative to this package and also \ diff --git a/stubs/paramiko/METADATA.toml b/stubs/paramiko/METADATA.toml index fc1fee0cd8cd..1c52e4c68492 100644 --- a/stubs/paramiko/METADATA.toml +++ b/stubs/paramiko/METADATA.toml @@ -1,5 +1,5 @@ version = "2.12.*" -requires = ["cryptography"] +requires = ["types-cryptography"] [tool.stubtest] # linux and darwin are equivalent diff --git a/stubs/requests/METADATA.toml b/stubs/requests/METADATA.toml index 571aac6a8f99..0fe1db2749cc 100644 --- a/stubs/requests/METADATA.toml +++ b/stubs/requests/METADATA.toml @@ -1,5 +1,5 @@ version = "2.28.*" -requires = ["types-urllib3<1.27", "numpy"] # keep in sync with requests's setup.py +requires = ["types-urllib3<1.27"] # keep in sync with requests's setup.py [tool.stubtest] ignore_missing_stub = false From 1c80dc1f182f2796d8f3275251910cd33259e5a8 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Dec 2022 18:09:22 +0000 Subject: [PATCH 06/17] Some small improvements --- requirements-tests.txt | 2 +- tests/mypy_test.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index af6c07358b46..889b7d79909a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -6,7 +6,7 @@ flake8-noqa==1.3.0 # must match .pre-commit-config.yaml flake8-pyi==22.11.0 # must match .pre-commit-config.yaml isort==5.11.3 # must match .pre-commit-config.yaml mypy==0.991 -packaging==21.3 +packaging==22.0 pathspec pycln==2.1.2 # must match .pre-commit-config.yaml pyyaml==6.0 diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 8172053f3ae8..a682a803faa1 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -24,6 +24,7 @@ from typing_extensions import Annotated, TypeAlias import tomli +from packaging.requirements import Requirement from utils import ( VERSIONS_RE as VERSION_LINE_RE, PackageDependencies, @@ -396,7 +397,11 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar for distribution_name, requirements in distributions.items(): if requirements.external_pkgs: num_pkgs_with_external_reqs += 1 - external_requirements_to_distributions[frozenset(requirements.external_pkgs)].append(distribution_name) + # convert to Requirement and then back to str + # to make sure that the requirements all have a normalised string representation + # (This will also catch any malformed requirements early) + external_requirements = frozenset(str(Requirement(pkg)) for pkg in requirements.external_pkgs) + external_requirements_to_distributions[external_requirements].append(distribution_name) else: _DISTRIBUTION_TO_VENV_MAPPING[distribution_name] = no_external_dependencies_venv @@ -462,7 +467,7 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe if not _DISTRIBUTION_TO_VENV_MAPPING: setup_virtual_environments(distributions_to_check, args, tempdir) - assert len(_DISTRIBUTION_TO_VENV_MAPPING) == len(distributions_to_check) + assert _DISTRIBUTION_TO_VENV_MAPPING.keys() == distributions_to_check.keys() for distribution, venv_info in _DISTRIBUTION_TO_VENV_MAPPING.items(): venv_python = venv_info.python_exe From e9355264fb65aeeb931424660b2016e1c4db5170 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 26 Dec 2022 18:20:10 +0000 Subject: [PATCH 07/17] Revert `packaging` bump --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 889b7d79909a..af6c07358b46 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -6,7 +6,7 @@ flake8-noqa==1.3.0 # must match .pre-commit-config.yaml flake8-pyi==22.11.0 # must match .pre-commit-config.yaml isort==5.11.3 # must match .pre-commit-config.yaml mypy==0.991 -packaging==22.0 +packaging==21.3 pathspec pycln==2.1.2 # must match .pre-commit-config.yaml pyyaml==6.0 From dd7a229b0d9d29578ad54a28c60542f4897e8430 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Tue, 27 Dec 2022 10:49:48 +0000 Subject: [PATCH 08/17] Move requirement-normalization logic to `utils.py` --- tests/mypy_test.py | 6 +----- tests/utils.py | 5 ++++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index a682a803faa1..78397ae73a60 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -24,7 +24,6 @@ from typing_extensions import Annotated, TypeAlias import tomli -from packaging.requirements import Requirement from utils import ( VERSIONS_RE as VERSION_LINE_RE, PackageDependencies, @@ -397,10 +396,7 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar for distribution_name, requirements in distributions.items(): if requirements.external_pkgs: num_pkgs_with_external_reqs += 1 - # convert to Requirement and then back to str - # to make sure that the requirements all have a normalised string representation - # (This will also catch any malformed requirements early) - external_requirements = frozenset(str(Requirement(pkg)) for pkg in requirements.external_pkgs) + external_requirements = frozenset(requirements.external_pkgs) external_requirements_to_distributions[external_requirements].append(distribution_name) else: _DISTRIBUTION_TO_VENV_MAPPING[distribution_name] = no_external_dependencies_venv diff --git a/tests/utils.py b/tests/utils.py index 98cc829db296..7b8974e5f7d9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -91,7 +91,10 @@ def read_dependencies(distribution: str) -> PackageDependencies: if maybe_typeshed_dependency in pypi_name_to_typeshed_name_mapping: typeshed.append(pypi_name_to_typeshed_name_mapping[maybe_typeshed_dependency]) else: - external.append(dependency) + # convert to Requirement and then back to str + # to make sure that the requirements all have a normalised string representation + # (This will also catch any malformed requirements early) + external.append(str(Requirement(dependency))) return PackageDependencies(tuple(typeshed), tuple(external)) From c5c93c920b613aabec4435916f721ae35339c0e1 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Tue, 27 Dec 2022 11:09:18 +0000 Subject: [PATCH 09/17] A better if-condition in `make_venv` --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 7b8974e5f7d9..be4b8c4918fc 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -145,7 +145,7 @@ def make_venv(venv_dir: Path) -> VenvInfo: try: venv.create(venv_dir, with_pip=True, clear=True) except subprocess.CalledProcessError as e: - if "ensurepip" in e.cmd and "KeyboardInterrupt" not in e.stdout: + if "ensurepip" in e.cmd and b"KeyboardInterrupt" not in e.stdout.splitlines(): print_error( "stubtest requires a Python installation with ensurepip. " "If on Linux, you may need to install the python3-venv package." From fec7b0b7e63f4d540884d36197791d19d8d6ac07 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Tue, 27 Dec 2022 14:13:23 +0000 Subject: [PATCH 10/17] Improve grammar in logged message, why not --- tests/mypy_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 78397ae73a60..0c83fe81b258 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -405,7 +405,11 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar if args.verbose: num_venvs = len(external_requirements_to_distributions) - msg = f"Setting up {num_venvs} venvs for {num_pkgs_with_external_reqs} distributions... " + msg = ( + f"Setting up {num_venvs} venv{'s' if num_venvs != 1 else ''} " + f"for {num_pkgs_with_external_reqs} " + f"distribution{'s' if num_pkgs_with_external_reqs != 1 else ''}... " + ) print(colored(msg, "blue"), end="", flush=True) venv_start_time = time.perf_counter() From fc085e1f9710052533c3c4e5fb786c0092b08e49 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 31 Dec 2022 21:47:57 +0000 Subject: [PATCH 11/17] Remove redundant error code --- tests/mypy_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 0c83fe81b258..e10d5a790cb3 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -238,7 +238,6 @@ def run_mypy( args.version, "--show-traceback", "--warn-incomplete-stub", - "--show-error-codes", "--no-error-summary", "--platform", args.platform, From a5902655f129a9ac6b6fdec45e49d174e55b9065 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 1 Jan 2023 00:34:00 +0000 Subject: [PATCH 12/17] Add issue number to comment, for easy greppability --- tests/mypy_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index e10d5a790cb3..9d8720e212e9 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -247,7 +247,7 @@ def run_mypy( # Stub completion is checked by pyright (--allow-*-defs) "--allow-untyped-defs", "--allow-incomplete-defs", - "--allow-subclassing-any", # TODO: Do we still need this now that non-types dependencies are allowed? + "--allow-subclassing-any", # TODO: Do we still need this now that non-types dependencies are allowed? (#5768) "--enable-error-code", "ignore-without-code", "--config-file", From 53c3b2201f6eacd9b50e42b628f8d1455817fbf0 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Tue, 3 Jan 2023 00:25:54 +0000 Subject: [PATCH 13/17] Run `pip freeze` if `--verbose` is specified, there are non-types dependencies, and the test failed --- tests/mypy_test.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 9d8720e212e9..025c98842906 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -219,7 +219,7 @@ def run_mypy( *, testing_stdlib: bool, non_types_dependencies: bool, - python_exe: str, + venv_info: VenvInfo, mypypath: str | None = None, ) -> ReturnCode: env_vars = dict(os.environ) @@ -259,7 +259,7 @@ def run_mypy( flags.append("--no-site-packages") mypy_args = [*flags, *map(str, files)] - mypy_command = [python_exe, "-m", "mypy"] + mypy_args + mypy_command = [venv_info.python_exe, "-m", "mypy"] + mypy_args if args.verbose: print(colored(f"running {' '.join(mypy_command)}", "blue")) result = subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars) @@ -269,6 +269,10 @@ def run_mypy( print_error(result.stdout) if result.stderr: print_error(result.stderr) + if non_types_dependencies and args.verbose: + print("Ran with the following environment:") + subprocess.run([venv_info.pip_exe, "freeze", "--all"]) + print() else: print_success_msg() return result.returncode @@ -307,7 +311,7 @@ class TestResults(NamedTuple): def test_third_party_distribution( - distribution: str, args: TestConfig, python_exe: str, *, non_types_dependencies: bool + distribution: str, args: TestConfig, venv_info: VenvInfo, *, non_types_dependencies: bool ) -> TestResults: """Test the stubs of a third-party distribution. @@ -336,7 +340,7 @@ def test_third_party_distribution( args, configurations, files, - python_exe=python_exe, + venv_info=venv_info, mypypath=mypypath, testing_stdlib=False, non_types_dependencies=non_types_dependencies, @@ -469,9 +473,9 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe assert _DISTRIBUTION_TO_VENV_MAPPING.keys() == distributions_to_check.keys() for distribution, venv_info in _DISTRIBUTION_TO_VENV_MAPPING.items(): - venv_python = venv_info.python_exe + non_types_dependencies = venv_info.python_exe != sys.executable this_code, checked = test_third_party_distribution( - distribution, args, python_exe=venv_python, non_types_dependencies=(venv_python != sys.executable) + distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies ) code = max(code, this_code) files_checked += checked From d5f05b87468d29174496ea6c360c108b25c63f1c Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Tue, 3 Jan 2023 10:38:40 +0000 Subject: [PATCH 14/17] Fix --- tests/mypy_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 025c98842906..ad79581e5561 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -348,7 +348,7 @@ def test_third_party_distribution( return TestResults(code, len(files)) -def test_stdlib(code: int, args: TestConfig) -> TestResults: +def test_stdlib(code: int, args: TestConfig, venv_info: VenvInfo) -> TestResults: files: list[Path] = [] stdlib = Path("stdlib") supported_versions = parse_versions(stdlib / "VERSIONS") @@ -362,7 +362,7 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults: if files: print(f"Testing stdlib ({len(files)} files)...", end="", flush=True) - this_code = run_mypy(args, [], files, python_exe=sys.executable, testing_stdlib=True, non_types_dependencies=False) + this_code = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False) code = max(code, this_code) return TestResults(code, len(files)) @@ -392,6 +392,7 @@ def install_requirements_for_venv(venv_info: VenvInfo, args: TestConfig, externa def setup_virtual_environments(distributions: dict[str, PackageDependencies], args: TestConfig, tempdir: Path) -> None: + # We don't actually need pip if there aren't any external dependencies no_external_dependencies_venv = VenvInfo(pip_exe="", python_exe=sys.executable) external_requirements_to_distributions: defaultdict[frozenset[str], list[str]] = defaultdict(list) num_pkgs_with_external_reqs = 0 @@ -488,7 +489,9 @@ def test_typeshed(code: int, args: TestConfig, tempdir: Path) -> TestResults: files_checked_this_version = 0 stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs") if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter): - code, stdlib_files_checked = test_stdlib(code, args) + # We don't actually need pip for the stdlib testing + stdlib_env = VenvInfo(pip_exe="", python_exe=sys.executable) + code, stdlib_files_checked = test_stdlib(code, args, venv_info=stdlib_env) files_checked_this_version += stdlib_files_checked print() From abbd4eb560ec0fa1b4b4542d62262ed19ef9c9cb Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Wed, 4 Jan 2023 12:27:22 +0000 Subject: [PATCH 15/17] Simplify --- tests/mypy_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index ad79581e5561..62fbe31688bd 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -348,7 +348,7 @@ def test_third_party_distribution( return TestResults(code, len(files)) -def test_stdlib(code: int, args: TestConfig, venv_info: VenvInfo) -> TestResults: +def test_stdlib(code: int, args: TestConfig) -> TestResults: files: list[Path] = [] stdlib = Path("stdlib") supported_versions = parse_versions(stdlib / "VERSIONS") @@ -362,6 +362,8 @@ def test_stdlib(code: int, args: TestConfig, venv_info: VenvInfo) -> TestResults if files: print(f"Testing stdlib ({len(files)} files)...", end="", flush=True) + # We don't actually need pip for the stdlib testing + venv_info = VenvInfo(pip_exe="", python_exe=sys.executable) this_code = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False) code = max(code, this_code) @@ -489,9 +491,7 @@ def test_typeshed(code: int, args: TestConfig, tempdir: Path) -> TestResults: files_checked_this_version = 0 stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs") if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter): - # We don't actually need pip for the stdlib testing - stdlib_env = VenvInfo(pip_exe="", python_exe=sys.executable) - code, stdlib_files_checked = test_stdlib(code, args, venv_info=stdlib_env) + code, stdlib_files_checked = test_stdlib(code, args) files_checked_this_version += stdlib_files_checked print() From 82c085c8f4b70d4b1f724b19a0de4a60a815a4e6 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Thu, 5 Jan 2023 10:20:30 +0000 Subject: [PATCH 16/17] Exit early from function if no stubs have external requirements --- tests/mypy_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 62fbe31688bd..d44afcd24bba 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -407,6 +407,11 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar else: _DISTRIBUTION_TO_VENV_MAPPING[distribution_name] = no_external_dependencies_venv + if num_pkgs_with_external_reqs == 0: + if args.verbose: + print(colored("No additional venvs are required to be set up", "blue")) + return + requirements_sets_to_venvs: dict[frozenset[str], VenvInfo] = {} if args.verbose: From 49b062c224eea203ef0ed8d8f7d93aad928ecec9 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Thu, 5 Jan 2023 20:48:03 +0000 Subject: [PATCH 17/17] Fix crash when a stubs package specifies a dependency on a non-types package and the dependency has a lower or upper bound --- tests/mypy_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index d44afcd24bba..f82b6739bda0 100644 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -375,8 +375,7 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults: def setup_venv_for_external_requirements_set(requirements_set: frozenset[str], tempdir: Path) -> tuple[frozenset[str], VenvInfo]: - reqs_joined = "-".join(sorted(requirements_set)) - venv_dir = tempdir / f".venv-{reqs_joined}" + venv_dir = tempdir / f".venv-{hash(requirements_set)}" return requirements_set, make_venv(venv_dir)