From e3d98b12db54054a7462ad134942c17f7c3a9c41 Mon Sep 17 00:00:00 2001 From: Avasam Date: Tue, 14 Feb 2023 04:51:07 -0500 Subject: [PATCH 01/10] Fix stdlib/disutils testing --- .github/workflows/daily.yml | 2 +- .github/workflows/stubtest_stdlib.yml | 2 +- tests/stubtest_stdlib.py | 117 ++++++++++++++++++-------- 3 files changed, 82 insertions(+), 39 deletions(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index e49787ba6298..11ca12658eee 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -45,7 +45,7 @@ jobs: cache: pip cache-dependency-path: requirements-tests.txt - name: Install dependencies - run: pip install $(grep mypy== requirements-tests.txt) + run: pip install typing_extensions $(grep ^pathspec requirements-tests.txt) - name: Run stubtest run: python tests/stubtest_stdlib.py diff --git a/.github/workflows/stubtest_stdlib.yml b/.github/workflows/stubtest_stdlib.yml index 3d0dd4a0d3ce..80738f6f0701 100644 --- a/.github/workflows/stubtest_stdlib.yml +++ b/.github/workflows/stubtest_stdlib.yml @@ -44,6 +44,6 @@ jobs: cache: pip cache-dependency-path: requirements-tests.txt - name: Install dependencies - run: pip install $(grep mypy== requirements-tests.txt) + run: pip install typing_extensions $(grep ^pathspec requirements-tests.txt) - name: Run stubtest run: python tests/stubtest_stdlib.py diff --git a/tests/stubtest_stdlib.py b/tests/stubtest_stdlib.py index c008353cd958..77c765de2a56 100755 --- a/tests/stubtest_stdlib.py +++ b/tests/stubtest_stdlib.py @@ -6,55 +6,98 @@ In typeshed CI, we run stubtest with each currently supported Python minor version. """ +from __future__ import annotations import subprocess import sys +import tempfile from pathlib import Path +from pkg_resources import parse_requirements + +from utils import make_venv, print_error + def run_stubtest(typeshed_dir: Path) -> int: allowlist_dir = typeshed_dir / "tests" / "stubtest_allowlists" version_allowlist = f"py{sys.version_info.major}{sys.version_info.minor}.txt" platform_allowlist = f"{sys.platform}.txt" combined_allowlist = f"{sys.platform}-py{sys.version_info.major}{sys.version_info.minor}.txt" + with tempfile.TemporaryDirectory() as tmp: + venv_dir = Path(tmp) + try: + pip_exe, python_exe = make_venv(venv_dir) + except Exception: + print_error("fail") + raise + + with Path("requirements-tests.txt").open() as requirements_txt: + requirements = [str(requirement) for requirement in parse_requirements(requirements_txt) if requirement.key == "mypy"] + pip_cmd = [pip_exe, "install"] + requirements + try: + subprocess.run(pip_cmd, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print_command_failure("Failed to install mypy", e) + return e.returncode + + # Uninstall setuptools from the venv so we can test stdlib's distutils + pip_cmd = [pip_exe, "uninstall", "-y", "setuptools"] + try: + subprocess.run(pip_cmd, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print_command_failure("Failed to uninstall setuptools", e) + return e.returncode + + cmd = [ + python_exe, + "-m", + "mypy.stubtest", + "--check-typeshed", + "--custom-typeshed-dir", + str(typeshed_dir), + "--allowlist", + str(allowlist_dir / "py3_common.txt"), + "--allowlist", + str(allowlist_dir / version_allowlist), + ] + if (allowlist_dir / platform_allowlist).exists(): + cmd += ["--allowlist", str(allowlist_dir / platform_allowlist)] + if (allowlist_dir / combined_allowlist).exists(): + cmd += ["--allowlist", str(allowlist_dir / combined_allowlist)] + if sys.version_info < (3, 10): + # As discussed in https://github.com/python/typeshed/issues/3693, we only aim for + # positional-only arg accuracy for python 3.10 and above. + cmd += ["--ignore-positional-only"] + print(" ".join(cmd), file=sys.stderr) + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + print( + "\nNB: stubtest output depends on the Python version (and system) it is run with. " + + "See README.md for more details.\n" + + "NB: We only check positional-only arg accuracy for Python 3.10.\n" + + f"\nCommand run was: {' '.join(cmd)}\n", + file=sys.stderr, + ) + print("\n\n", file=sys.stderr) + print(f'To fix "unused allowlist" errors, remove the corresponding entries from {allowlist_dir}', file=sys.stderr) + return e.returncode + else: + print("stubtest succeeded", file=sys.stderr) + return 0 + + +def print_command_failure(message: str, e: subprocess.CalledProcessError) -> None: + print_error("fail") + print(file=sys.stderr) + print(message, file=sys.stderr) + print_command_output(e) + - cmd = [ - sys.executable, - "-m", - "mypy.stubtest", - "--check-typeshed", - "--custom-typeshed-dir", - str(typeshed_dir), - "--allowlist", - str(allowlist_dir / "py3_common.txt"), - "--allowlist", - str(allowlist_dir / version_allowlist), - ] - if (allowlist_dir / platform_allowlist).exists(): - cmd += ["--allowlist", str(allowlist_dir / platform_allowlist)] - if (allowlist_dir / combined_allowlist).exists(): - cmd += ["--allowlist", str(allowlist_dir / combined_allowlist)] - if sys.version_info < (3, 10): - # As discussed in https://github.com/python/typeshed/issues/3693, we only aim for - # positional-only arg accuracy for the latest Python version. - cmd += ["--ignore-positional-only"] - print(" ".join(cmd), file=sys.stderr) - try: - subprocess.run(cmd, check=True) - except subprocess.CalledProcessError as e: - print( - "\nNB: stubtest output depends on the Python version (and system) it is run with. " - "See README.md for more details.\n" - "NB: We only check positional-only arg accuracy for Python 3.10.\n" - f"\nCommand run was: {' '.join(cmd)}\n", - file=sys.stderr, - ) - print("\n\n", file=sys.stderr) - print(f'To fix "unused allowlist" errors, remove the corresponding entries from {allowlist_dir}', file=sys.stderr) - return e.returncode - else: - print("stubtest succeeded", file=sys.stderr) - return 0 +def print_command_output(e: subprocess.CalledProcessError | subprocess.CompletedProcess[bytes]) -> None: + print(e.stdout.decode(), end="", file=sys.stderr) + print(e.stderr.decode(), end="", file=sys.stderr) + print(file=sys.stderr) if __name__ == "__main__": From f7c89860430d76c4606e5f5f85d76345f7b84759 Mon Sep 17 00:00:00 2001 From: Avasam Date: Tue, 14 Feb 2023 04:51:21 -0500 Subject: [PATCH 02/10] Turn on testing in python 3.10+ --- tests/stubtest_allowlists/py310.txt | 5 ----- tests/stubtest_allowlists/py311.txt | 4 ---- 2 files changed, 9 deletions(-) diff --git a/tests/stubtest_allowlists/py310.txt b/tests/stubtest_allowlists/py310.txt index a3c77a1b91e1..d03248836278 100644 --- a/tests/stubtest_allowlists/py310.txt +++ b/tests/stubtest_allowlists/py310.txt @@ -85,7 +85,6 @@ _collections_abc.MappingView.__class_getitem__ _csv.Reader _csv.Writer bdb.Breakpoint.clearBreakpoints -distutils.util.get_host_platform inspect.Signature.from_builtin # Removed in 3.11, can add if someone needs this inspect.Signature.from_function # Removed in 3.11, can add if someone needs this multiprocessing.managers.SharedMemoryServer.create @@ -168,7 +167,3 @@ ast.ImportFrom.level # None on the class, but never None on instances # White lies around defaults dataclasses.KW_ONLY - -# stubtest confuses stdlib distutils with setuptools-bundled distutils (#8410), -# and the whole directory is going to be removed in 3.12 anyway -distutils\..* diff --git a/tests/stubtest_allowlists/py311.txt b/tests/stubtest_allowlists/py311.txt index 12adfda655ca..56651b8de9fc 100644 --- a/tests/stubtest_allowlists/py311.txt +++ b/tests/stubtest_allowlists/py311.txt @@ -138,7 +138,3 @@ typing._TypedDict.values # White lies around defaults dataclasses.KW_ONLY - -# stubtest confuses stdlib distutils with setuptools-bundled distutils (#8410), -# and the whole directory is going to be removed in 3.12 anyway -distutils\..* From a8cf9681dc4c5d316b76d7ad9ca661ffa245f89a Mon Sep 17 00:00:00 2001 From: Avasam Date: Tue, 14 Feb 2023 04:51:36 -0500 Subject: [PATCH 03/10] Fix tests --- pyrightconfig.stricter.json | 1 + stdlib/distutils/cmd.pyi | 3 ++ stdlib/distutils/core.pyi | 11 ++++- stdlib/distutils/cygwinccompiler.pyi | 16 ++++++ stdlib/distutils/dist.pyi | 58 +++++++++++++++++++++- stdlib/distutils/fancy_getopt.pyi | 18 +++++-- stdlib/distutils/sysconfig.pyi | 9 ++++ stdlib/distutils/util.pyi | 4 ++ tests/stubtest_allowlists/py37.txt | 61 ----------------------- tests/stubtest_allowlists/py38.txt | 62 ------------------------ tests/stubtest_allowlists/py39.txt | 62 ------------------------ tests/stubtest_allowlists/py3_common.txt | 28 ----------- 12 files changed, 114 insertions(+), 219 deletions(-) diff --git a/pyrightconfig.stricter.json b/pyrightconfig.stricter.json index 80f5e6f6248a..9125ba43ddeb 100644 --- a/pyrightconfig.stricter.json +++ b/pyrightconfig.stricter.json @@ -9,6 +9,7 @@ // test cases use a custom pyrightconfig file "stubs/**/@tests/test_cases", "stdlib/distutils/command", + "stdlib/distutils/dist.pyi", "stdlib/lib2to3/refactor.pyi", "stdlib/_tkinter.pyi", "stdlib/tkinter/__init__.pyi", diff --git a/stdlib/distutils/cmd.pyi b/stdlib/distutils/cmd.pyi index d9ffee9cb832..d5e04c7dc034 100644 --- a/stdlib/distutils/cmd.pyi +++ b/stdlib/distutils/cmd.pyi @@ -1,3 +1,4 @@ +from _typeshed import Incomplete from abc import abstractmethod from collections.abc import Callable, Iterable from distutils.dist import Distribution @@ -60,3 +61,5 @@ class Command: skip_msg: str | None = None, level: Any = 1, ) -> None: ... # level is not used + def ensure_finalized(self) -> None: ... + def dump_options(self, header: Incomplete | None = ..., indent: str = ...) -> None: ... diff --git a/stdlib/distutils/core.pyi b/stdlib/distutils/core.pyi index 56081f921378..927e88540b89 100644 --- a/stdlib/distutils/core.pyi +++ b/stdlib/distutils/core.pyi @@ -1,8 +1,17 @@ +from _typeshed import Incomplete from collections.abc import Mapping from distutils.cmd import Command as Command from distutils.dist import Distribution as Distribution from distutils.extension import Extension as Extension -from typing import Any +from os import PathLike +from typing import Any, AnyStr + +USAGE: str + +def gen_usage(script_name: PathLike[AnyStr]) -> str: ... + +setup_keywords: Incomplete +extension_keywords: Incomplete def setup( *, diff --git a/stdlib/distutils/cygwinccompiler.pyi b/stdlib/distutils/cygwinccompiler.pyi index 1f85b254860b..36740fcb0422 100644 --- a/stdlib/distutils/cygwinccompiler.pyi +++ b/stdlib/distutils/cygwinccompiler.pyi @@ -1,4 +1,20 @@ +from _typeshed import Incomplete from distutils.unixccompiler import UnixCCompiler +from distutils.version import LooseVersion +from typing_extensions import Literal + +def get_msvcr() -> list[str] | None: ... class CygwinCCompiler(UnixCCompiler): ... class Mingw32CCompiler(CygwinCCompiler): ... + +CONFIG_H_OK: str +CONFIG_H_NOTOK: str +CONFIG_H_UNCERTAIN: str + +def check_config_h() -> tuple[Literal["ok", "not ok", "uncertain"], str]: ... + +RE_VERSION: Incomplete + +def get_versions() -> tuple[LooseVersion | None, ...]: ... +def is_cygwingcc() -> bool: ... diff --git a/stdlib/distutils/dist.pyi b/stdlib/distutils/dist.pyi index b411324c4ce6..55ba06a58f69 100644 --- a/stdlib/distutils/dist.pyi +++ b/stdlib/distutils/dist.pyi @@ -1,8 +1,10 @@ -from _typeshed import FileDescriptorOrPath, SupportsWrite +from _typeshed import FileDescriptorOrPath, Incomplete, SupportsWrite from collections.abc import Iterable, Mapping from distutils.cmd import Command from typing import IO, Any +command_re: Incomplete + class DistributionMetadata: def __init__(self, path: FileDescriptorOrPath | None = None) -> None: ... name: str | None @@ -57,3 +59,57 @@ class Distribution: def get_option_dict(self, command: str) -> dict[str, tuple[str, str]]: ... def parse_config_files(self, filenames: Iterable[str] | None = None) -> None: ... def get_command_obj(self, command: str, create: bool = ...) -> Command | None: ... + global_options: Incomplete + common_usage: str + display_options: Incomplete + display_option_names: Incomplete + negative_opt: Incomplete + verbose: int + dry_run: int + help: int + command_packages: Incomplete + script_name: Incomplete + script_args: Incomplete + command_options: Incomplete + dist_files: Incomplete + packages: Incomplete + package_data: Incomplete + package_dir: Incomplete + py_modules: Incomplete + libraries: Incomplete + headers: Incomplete + ext_modules: Incomplete + ext_package: Incomplete + include_dirs: Incomplete + extra_path: Incomplete + scripts: Incomplete + data_files: Incomplete + password: str + command_obj: Incomplete + have_run: Incomplete + want_user_cfg: bool + def dump_option_dicts( + self, header: Incomplete | None = ..., commands: Incomplete | None = ..., indent: str = ... + ) -> None: ... + def find_config_files(self): ... + commands: Incomplete + def parse_command_line(self): ... + def finalize_options(self) -> None: ... + def handle_display_options(self, option_order): ... + def print_command_list(self, commands, header, max_length) -> None: ... + def print_commands(self) -> None: ... + def get_command_list(self): ... + def get_command_packages(self): ... + def get_command_class(self, command): ... + def reinitialize_command(self, command, reinit_subcommands: int = ...): ... + def announce(self, msg, level=...) -> None: ... + def run_commands(self) -> None: ... + def run_command(self, command) -> None: ... + def has_pure_modules(self): ... + def has_ext_modules(self): ... + def has_c_libraries(self): ... + def has_modules(self): ... + def has_headers(self): ... + def has_scripts(self): ... + def has_data_files(self): ... + def is_pure(self): ... diff --git a/stdlib/distutils/fancy_getopt.pyi b/stdlib/distutils/fancy_getopt.pyi index 153583be6b5d..1d6566f33f0d 100644 --- a/stdlib/distutils/fancy_getopt.pyi +++ b/stdlib/distutils/fancy_getopt.pyi @@ -1,3 +1,4 @@ +from _typeshed import Incomplete from collections.abc import Iterable, Mapping from typing import Any, overload from typing_extensions import TypeAlias @@ -5,10 +6,10 @@ from typing_extensions import TypeAlias _Option: TypeAlias = tuple[str, str | None, str] _GR: TypeAlias = tuple[list[str], OptionDummy] -def fancy_getopt( - options: list[_Option], negative_opt: Mapping[_Option, _Option], object: Any, args: list[str] | None -) -> list[str] | _GR: ... -def wrap_text(text: str, width: int) -> list[str]: ... +longopt_pat: str +longopt_re: Incomplete +neg_alias_re: Incomplete +longopt_xlate: Incomplete class FancyGetopt: def __init__(self, option_table: list[_Option] | None = None) -> None: ... @@ -20,5 +21,14 @@ class FancyGetopt: def get_option_order(self) -> list[tuple[str, str]]: ... def generate_help(self, header: str | None = None) -> list[str]: ... +def fancy_getopt( + options: list[_Option], negative_opt: Mapping[_Option, _Option], object: Any, args: list[str] | None +) -> list[str] | _GR: ... + +WS_TRANS: Incomplete + +def wrap_text(text: str, width: int) -> list[str]: ... +def translate_longopt(opt: str) -> str: ... + class OptionDummy: def __init__(self, options: Iterable[str] = ...) -> None: ... diff --git a/stdlib/distutils/sysconfig.pyi b/stdlib/distutils/sysconfig.pyi index 8b291e8b94a5..464cfb639c6d 100644 --- a/stdlib/distutils/sysconfig.pyi +++ b/stdlib/distutils/sysconfig.pyi @@ -1,9 +1,15 @@ +import sys from collections.abc import Mapping from distutils.ccompiler import CCompiler PREFIX: str EXEC_PREFIX: str +BASE_PREFIX: str +BASE_EXEC_PREFIX: str +project_base: str +python_build: bool +def expand_makefile_vars(s: str, vars: Mapping[str, str]) -> str: ... def get_config_var(name: str) -> int | str | None: ... def get_config_vars(*args: str) -> Mapping[str, int | str]: ... def get_config_h_filename() -> str: ... @@ -11,3 +17,6 @@ def get_makefile_filename() -> str: ... def get_python_inc(plat_specific: bool = ..., prefix: str | None = None) -> str: ... def get_python_lib(plat_specific: bool = ..., standard_lib: bool = ..., prefix: str | None = None) -> str: ... def customize_compiler(compiler: CCompiler) -> None: ... + +if sys.version_info < (3, 10): + def get_python_version() -> str: ... diff --git a/stdlib/distutils/util.pyi b/stdlib/distutils/util.pyi index f03844307581..83b03747fda6 100644 --- a/stdlib/distutils/util.pyi +++ b/stdlib/distutils/util.pyi @@ -1,8 +1,12 @@ +import sys from _typeshed import StrPath, Unused from collections.abc import Callable, Container, Iterable, Mapping from typing import Any from typing_extensions import Literal +if sys.version_info >= (3, 8): + def get_host_platform() -> str: ... + def get_platform() -> str: ... def convert_path(pathname: str) -> str: ... def change_root(new_root: str, pathname: str) -> str: ... diff --git a/tests/stubtest_allowlists/py37.txt b/tests/stubtest_allowlists/py37.txt index 78e7adbd291d..f770b8075666 100644 --- a/tests/stubtest_allowlists/py37.txt +++ b/tests/stubtest_allowlists/py37.txt @@ -74,16 +74,6 @@ tkinter.Tk.__init__ # Exists at runtime, but missing from stubs contextvars.ContextVar.__class_getitem__ datetime.datetime_CAPI -distutils.sysconfig.expand_makefile_vars -distutils.sysconfig.get_python_version -distutils.cygwinccompiler.RE_VERSION -distutils.dist.command_re -distutils.fancy_getopt.longopt_re -distutils.fancy_getopt.neg_alias_re -distutils.core.USAGE -distutils.core.extension_keywords -distutils.core.gen_usage -distutils.core.setup_keywords dummy_threading.Lock dummy_threading.RLock html.parser.HTMLParser.unescape @@ -156,54 +146,3 @@ builtins.property.__get__ builtins.staticmethod.__get__ types.FunctionType.__get__ types.LambdaType.__get__ - -# Missing from distutils (deprecated, to be removed in 3.12) -distutils.core.USAGE -distutils.core.extension_keywords -distutils.core.gen_usage -distutils.core.setup_keywords -distutils.core.Command.dump_options -distutils.core.Command.ensure_finalized -distutils.core.Distribution.announce -distutils.core.Distribution.common_usage -distutils.core.Distribution.display_option_names -distutils.core.Distribution.display_options -distutils.core.Distribution.dump_option_dicts -distutils.core.Distribution.find_config_files -distutils.core.Distribution.get_command_packages -distutils.core.Distribution.global_options -distutils.core.Distribution.has_c_libraries -distutils.core.Distribution.has_data_files -distutils.core.Distribution.has_ext_modules -distutils.core.Distribution.has_headers -distutils.core.Distribution.has_modules -distutils.core.Distribution.has_pure_modules -distutils.core.Distribution.has_scripts -distutils.core.Distribution.is_pure -distutils.core.Distribution.negative_opt -distutils.core.Distribution.parse_command_line -distutils.core.Distribution.print_command_list -distutils.core.Distribution.reinitialize_command -distutils.core.Distribution.run_commands -distutils.cygwinccompiler.is_cygwingcc -distutils.dist.Distribution.announce -distutils.dist.Distribution.common_usage -distutils.dist.Distribution.display_option_names -distutils.dist.Distribution.display_options -distutils.dist.Distribution.dump_option_dicts -distutils.dist.Distribution.find_config_files -distutils.dist.Distribution.get_command_packages -distutils.dist.Distribution.global_options -distutils.dist.Distribution.has_c_libraries -distutils.dist.Distribution.has_data_files -distutils.dist.Distribution.has_ext_modules -distutils.dist.Distribution.has_headers -distutils.dist.Distribution.has_modules -distutils.dist.Distribution.has_pure_modules -distutils.dist.Distribution.has_scripts -distutils.dist.Distribution.is_pure -distutils.dist.Distribution.negative_opt -distutils.dist.Distribution.parse_command_line -distutils.dist.Distribution.print_command_list -distutils.dist.Distribution.reinitialize_command -distutils.dist.Distribution.run_commands diff --git a/tests/stubtest_allowlists/py38.txt b/tests/stubtest_allowlists/py38.txt index e9667438e5f1..8b9cfffc3ab5 100644 --- a/tests/stubtest_allowlists/py38.txt +++ b/tests/stubtest_allowlists/py38.txt @@ -34,10 +34,6 @@ collections.KeysView.__reversed__ collections.ValuesView.__reversed__ collections.Mapping.__reversed__ # Set to None at runtime for a better error message distutils.command.bdist_wininst # see #6523 -distutils.core.USAGE -distutils.core.extension_keywords -distutils.core.gen_usage -distutils.core.setup_keywords dummy_threading.Condition.acquire dummy_threading.Condition.release dummy_threading.Event.isSet @@ -95,13 +91,6 @@ tkinter.Tk.__init__ # Exists at runtime, but missing from stubs contextvars.ContextVar.__class_getitem__ datetime.datetime_CAPI -distutils.sysconfig.expand_makefile_vars -distutils.sysconfig.get_python_version -distutils.util.get_host_platform -distutils.cygwinccompiler.RE_VERSION -distutils.dist.command_re -distutils.fancy_getopt.longopt_re -distutils.fancy_getopt.neg_alias_re dummy_threading.ExceptHookArgs dummy_threading.Lock dummy_threading.RLock @@ -173,54 +162,3 @@ builtins.property.__get__ builtins.staticmethod.__get__ types.FunctionType.__get__ types.LambdaType.__get__ - -# Missing from distutils (deprecated, to be removed in 3.12) -distutils.core.USAGE -distutils.core.extension_keywords -distutils.core.gen_usage -distutils.core.setup_keywords -distutils.core.Command.dump_options -distutils.core.Command.ensure_finalized -distutils.core.Distribution.announce -distutils.core.Distribution.common_usage -distutils.core.Distribution.display_option_names -distutils.core.Distribution.display_options -distutils.core.Distribution.dump_option_dicts -distutils.core.Distribution.find_config_files -distutils.core.Distribution.get_command_packages -distutils.core.Distribution.global_options -distutils.core.Distribution.has_c_libraries -distutils.core.Distribution.has_data_files -distutils.core.Distribution.has_ext_modules -distutils.core.Distribution.has_headers -distutils.core.Distribution.has_modules -distutils.core.Distribution.has_pure_modules -distutils.core.Distribution.has_scripts -distutils.core.Distribution.is_pure -distutils.core.Distribution.negative_opt -distutils.core.Distribution.parse_command_line -distutils.core.Distribution.print_command_list -distutils.core.Distribution.reinitialize_command -distutils.core.Distribution.run_commands -distutils.cygwinccompiler.is_cygwingcc -distutils.dist.Distribution.announce -distutils.dist.Distribution.common_usage -distutils.dist.Distribution.display_option_names -distutils.dist.Distribution.display_options -distutils.dist.Distribution.dump_option_dicts -distutils.dist.Distribution.find_config_files -distutils.dist.Distribution.get_command_packages -distutils.dist.Distribution.global_options -distutils.dist.Distribution.has_c_libraries -distutils.dist.Distribution.has_data_files -distutils.dist.Distribution.has_ext_modules -distutils.dist.Distribution.has_headers -distutils.dist.Distribution.has_modules -distutils.dist.Distribution.has_pure_modules -distutils.dist.Distribution.has_scripts -distutils.dist.Distribution.is_pure -distutils.dist.Distribution.negative_opt -distutils.dist.Distribution.parse_command_line -distutils.dist.Distribution.print_command_list -distutils.dist.Distribution.reinitialize_command -distutils.dist.Distribution.run_commands diff --git a/tests/stubtest_allowlists/py39.txt b/tests/stubtest_allowlists/py39.txt index 54b0f0813798..c967bbcabc1e 100644 --- a/tests/stubtest_allowlists/py39.txt +++ b/tests/stubtest_allowlists/py39.txt @@ -34,10 +34,6 @@ collections.ValuesView.__reversed__ contextlib.AbstractAsyncContextManager.__class_getitem__ contextlib.AbstractContextManager.__class_getitem__ distutils.command.bdist_wininst # see #6523 -distutils.core.USAGE -distutils.core.extension_keywords -distutils.core.gen_usage -distutils.core.setup_keywords fractions.Fraction.__new__ # overload is too complicated for stubtest to resolve gettext.install gettext.translation @@ -95,13 +91,6 @@ collections.Awaitable.__class_getitem__ collections.Container.__class_getitem__ collections.Iterable.__class_getitem__ collections.MappingView.__class_getitem__ -distutils.sysconfig.expand_makefile_vars -distutils.sysconfig.get_python_version -distutils.util.get_host_platform -distutils.cygwinccompiler.RE_VERSION -distutils.dist.command_re -distutils.fancy_getopt.longopt_re -distutils.fancy_getopt.neg_alias_re hmac.HMAC.digest_cons hmac.HMAC.inner hmac.HMAC.outer @@ -168,54 +157,3 @@ builtins.property.__get__ builtins.staticmethod.__get__ types.FunctionType.__get__ types.LambdaType.__get__ - -# Missing from distutils (deprecated, to be removed in 3.12) -distutils.core.USAGE -distutils.core.extension_keywords -distutils.core.gen_usage -distutils.core.setup_keywords -distutils.core.Command.dump_options -distutils.core.Command.ensure_finalized -distutils.core.Distribution.announce -distutils.core.Distribution.common_usage -distutils.core.Distribution.display_option_names -distutils.core.Distribution.display_options -distutils.core.Distribution.dump_option_dicts -distutils.core.Distribution.find_config_files -distutils.core.Distribution.get_command_packages -distutils.core.Distribution.global_options -distutils.core.Distribution.has_c_libraries -distutils.core.Distribution.has_data_files -distutils.core.Distribution.has_ext_modules -distutils.core.Distribution.has_headers -distutils.core.Distribution.has_modules -distutils.core.Distribution.has_pure_modules -distutils.core.Distribution.has_scripts -distutils.core.Distribution.is_pure -distutils.core.Distribution.negative_opt -distutils.core.Distribution.parse_command_line -distutils.core.Distribution.print_command_list -distutils.core.Distribution.reinitialize_command -distutils.core.Distribution.run_commands -distutils.cygwinccompiler.is_cygwingcc -distutils.dist.Distribution.announce -distutils.dist.Distribution.common_usage -distutils.dist.Distribution.display_option_names -distutils.dist.Distribution.display_options -distutils.dist.Distribution.dump_option_dicts -distutils.dist.Distribution.find_config_files -distutils.dist.Distribution.get_command_packages -distutils.dist.Distribution.global_options -distutils.dist.Distribution.has_c_libraries -distutils.dist.Distribution.has_data_files -distutils.dist.Distribution.has_ext_modules -distutils.dist.Distribution.has_headers -distutils.dist.Distribution.has_modules -distutils.dist.Distribution.has_pure_modules -distutils.dist.Distribution.has_scripts -distutils.dist.Distribution.is_pure -distutils.dist.Distribution.negative_opt -distutils.dist.Distribution.parse_command_line -distutils.dist.Distribution.print_command_list -distutils.dist.Distribution.reinitialize_command -distutils.dist.Distribution.run_commands diff --git a/tests/stubtest_allowlists/py3_common.txt b/tests/stubtest_allowlists/py3_common.txt index a035c9ad7bf5..ad3f61293e34 100644 --- a/tests/stubtest_allowlists/py3_common.txt +++ b/tests/stubtest_allowlists/py3_common.txt @@ -643,19 +643,8 @@ distutils.ccompiler.CCompiler.src_extensions distutils.ccompiler.CCompiler.static_lib_extension distutils.ccompiler.CCompiler.static_lib_format distutils.ccompiler.compiler_class -distutils.cmd.Command.dump_options -distutils.cmd.Command.ensure_finalized distutils.command.bdist distutils.command.install.* -distutils.core.Distribution.finalize_options -distutils.core.Distribution.get_command_class -distutils.core.Distribution.get_command_list -distutils.core.Distribution.handle_display_options -distutils.core.Distribution.print_commands -distutils.core.Distribution.run_command -distutils.cygwinccompiler.CONFIG_H_NOTOK -distutils.cygwinccompiler.CONFIG_H_OK -distutils.cygwinccompiler.CONFIG_H_UNCERTAIN distutils.cygwinccompiler.CygwinCCompiler.compiler_type distutils.cygwinccompiler.CygwinCCompiler.exe_extension distutils.cygwinccompiler.CygwinCCompiler.obj_extension @@ -664,22 +653,12 @@ distutils.cygwinccompiler.CygwinCCompiler.shared_lib_format distutils.cygwinccompiler.CygwinCCompiler.static_lib_extension distutils.cygwinccompiler.CygwinCCompiler.static_lib_format distutils.cygwinccompiler.Mingw32CCompiler.compiler_type -distutils.cygwinccompiler.check_config_h -distutils.cygwinccompiler.get_msvcr -distutils.cygwinccompiler.get_versions distutils.dir_util.ensure_relative -distutils.dist.Distribution.finalize_options -distutils.dist.Distribution.get_command_class -distutils.dist.Distribution.get_command_list -distutils.dist.Distribution.handle_display_options -distutils.dist.Distribution.print_commands -distutils.dist.Distribution.run_command distutils.dist.DistributionMetadata.set_classifiers distutils.dist.DistributionMetadata.set_keywords distutils.dist.DistributionMetadata.set_platforms distutils.dist.fix_help_options distutils.extension.read_setup_file -distutils.fancy_getopt.WS_TRANS distutils.fancy_getopt.FancyGetopt.add_option distutils.fancy_getopt.FancyGetopt.get_attr_name distutils.fancy_getopt.FancyGetopt.has_option @@ -687,9 +666,6 @@ distutils.fancy_getopt.FancyGetopt.print_help distutils.fancy_getopt.FancyGetopt.set_aliases distutils.fancy_getopt.FancyGetopt.set_negative_aliases distutils.fancy_getopt.FancyGetopt.set_option_table -distutils.fancy_getopt.longopt_pat -distutils.fancy_getopt.longopt_xlate -distutils.fancy_getopt.translate_longopt distutils.msvccompiler.MSVCCompiler.compiler_type distutils.msvccompiler.MSVCCompiler.exe_extension distutils.msvccompiler.MSVCCompiler.executables @@ -708,13 +684,9 @@ distutils.msvccompiler.get_build_version distutils.msvccompiler.normalize_and_reduce_paths distutils.msvccompiler.read_keys distutils.msvccompiler.read_values -distutils.sysconfig.BASE_EXEC_PREFIX -distutils.sysconfig.BASE_PREFIX distutils.sysconfig.build_flags distutils.sysconfig.parse_config_h distutils.sysconfig.parse_makefile -distutils.sysconfig.project_base -distutils.sysconfig.python_build distutils.text_file.TextFile.default_options distutils.text_file.TextFile.error distutils.text_file.TextFile.gen_error From c6a5e030595be81049d9ce2cf35dd449d5a6cc91 Mon Sep 17 00:00:00 2001 From: Avasam Date: Sun, 19 Feb 2023 12:37:36 -0500 Subject: [PATCH 04/10] Update .github/workflows/daily.yml Co-authored-by: Alex Waygood --- .github/workflows/daily.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 11ca12658eee..99234d5de7ba 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -45,7 +45,7 @@ jobs: cache: pip cache-dependency-path: requirements-tests.txt - name: Install dependencies - run: pip install typing_extensions $(grep ^pathspec requirements-tests.txt) + run: pip install -r requirements-tests.txt - name: Run stubtest run: python tests/stubtest_stdlib.py From b33c1ab7ce81f0906055368588eb2508981109b6 Mon Sep 17 00:00:00 2001 From: Avasam Date: Sun, 19 Feb 2023 12:39:12 -0500 Subject: [PATCH 05/10] Apply suggestions from code review Co-authored-by: Alex Waygood --- .github/workflows/stubtest_stdlib.yml | 2 +- stdlib/distutils/cmd.pyi | 2 +- stdlib/distutils/core.pyi | 6 +++--- stdlib/distutils/cygwinccompiler.pyi | 2 +- stdlib/distutils/dist.pyi | 2 +- stdlib/distutils/fancy_getopt.pyi | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/stubtest_stdlib.yml b/.github/workflows/stubtest_stdlib.yml index 80738f6f0701..a43e0925a1e8 100644 --- a/.github/workflows/stubtest_stdlib.yml +++ b/.github/workflows/stubtest_stdlib.yml @@ -44,6 +44,6 @@ jobs: cache: pip cache-dependency-path: requirements-tests.txt - name: Install dependencies - run: pip install typing_extensions $(grep ^pathspec requirements-tests.txt) + run: pip install -r requirements-tests.txt - name: Run stubtest run: python tests/stubtest_stdlib.py diff --git a/stdlib/distutils/cmd.pyi b/stdlib/distutils/cmd.pyi index d5e04c7dc034..a9aade0206dd 100644 --- a/stdlib/distutils/cmd.pyi +++ b/stdlib/distutils/cmd.pyi @@ -62,4 +62,4 @@ class Command: level: Any = 1, ) -> None: ... # level is not used def ensure_finalized(self) -> None: ... - def dump_options(self, header: Incomplete | None = ..., indent: str = ...) -> None: ... + def dump_options(self, header: Incomplete | None = None, indent: str = "") -> None: ... diff --git a/stdlib/distutils/core.pyi b/stdlib/distutils/core.pyi index 927e88540b89..f633170e1537 100644 --- a/stdlib/distutils/core.pyi +++ b/stdlib/distutils/core.pyi @@ -8,10 +8,10 @@ from typing import Any, AnyStr USAGE: str -def gen_usage(script_name: PathLike[AnyStr]) -> str: ... +def gen_usage(script_name: StrOrBytesPath) -> str: ... -setup_keywords: Incomplete -extension_keywords: Incomplete +setup_keywords: tuple[str, ...] +extension_keywords: tuple[str, ...] def setup( *, diff --git a/stdlib/distutils/cygwinccompiler.pyi b/stdlib/distutils/cygwinccompiler.pyi index 36740fcb0422..8bb86cb04e24 100644 --- a/stdlib/distutils/cygwinccompiler.pyi +++ b/stdlib/distutils/cygwinccompiler.pyi @@ -14,7 +14,7 @@ CONFIG_H_UNCERTAIN: str def check_config_h() -> tuple[Literal["ok", "not ok", "uncertain"], str]: ... -RE_VERSION: Incomplete +RE_VERSION: re.Pattern[bytes] def get_versions() -> tuple[LooseVersion | None, ...]: ... def is_cygwingcc() -> bool: ... diff --git a/stdlib/distutils/dist.pyi b/stdlib/distutils/dist.pyi index 55ba06a58f69..0b9f80e11bdc 100644 --- a/stdlib/distutils/dist.pyi +++ b/stdlib/distutils/dist.pyi @@ -3,7 +3,7 @@ from collections.abc import Iterable, Mapping from distutils.cmd import Command from typing import IO, Any -command_re: Incomplete +command_re: re.Pattern[str] class DistributionMetadata: def __init__(self, path: FileDescriptorOrPath | None = None) -> None: ... diff --git a/stdlib/distutils/fancy_getopt.pyi b/stdlib/distutils/fancy_getopt.pyi index 1d6566f33f0d..c2ac15ced979 100644 --- a/stdlib/distutils/fancy_getopt.pyi +++ b/stdlib/distutils/fancy_getopt.pyi @@ -7,9 +7,9 @@ _Option: TypeAlias = tuple[str, str | None, str] _GR: TypeAlias = tuple[list[str], OptionDummy] longopt_pat: str -longopt_re: Incomplete -neg_alias_re: Incomplete -longopt_xlate: Incomplete +longopt_re: re.Pattern[str] +neg_alias_re: re.Pattern[str] +longopt_xlate: dict[int, int] class FancyGetopt: def __init__(self, option_table: list[_Option] | None = None) -> None: ... @@ -25,7 +25,7 @@ def fancy_getopt( options: list[_Option], negative_opt: Mapping[_Option, _Option], object: Any, args: list[str] | None ) -> list[str] | _GR: ... -WS_TRANS: Incomplete +WS_TRANS: dict[int, str] def wrap_text(text: str, width: int) -> list[str]: ... def translate_longopt(opt: str) -> str: ... From b199d150b529934b8d6703a93d348c3cf7b30fd5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 19 Feb 2023 17:40:18 +0000 Subject: [PATCH 06/10] [pre-commit.ci] auto fixes from pre-commit.com hooks --- stdlib/distutils/core.pyi | 4 +--- stdlib/distutils/cygwinccompiler.pyi | 1 - stdlib/distutils/fancy_getopt.pyi | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/stdlib/distutils/core.pyi b/stdlib/distutils/core.pyi index f633170e1537..7bef94932beb 100644 --- a/stdlib/distutils/core.pyi +++ b/stdlib/distutils/core.pyi @@ -1,10 +1,8 @@ -from _typeshed import Incomplete from collections.abc import Mapping from distutils.cmd import Command as Command from distutils.dist import Distribution as Distribution from distutils.extension import Extension as Extension -from os import PathLike -from typing import Any, AnyStr +from typing import Any USAGE: str diff --git a/stdlib/distutils/cygwinccompiler.pyi b/stdlib/distutils/cygwinccompiler.pyi index 8bb86cb04e24..b45fdbbceae6 100644 --- a/stdlib/distutils/cygwinccompiler.pyi +++ b/stdlib/distutils/cygwinccompiler.pyi @@ -1,4 +1,3 @@ -from _typeshed import Incomplete from distutils.unixccompiler import UnixCCompiler from distutils.version import LooseVersion from typing_extensions import Literal diff --git a/stdlib/distutils/fancy_getopt.pyi b/stdlib/distutils/fancy_getopt.pyi index c2ac15ced979..fec289ac45a5 100644 --- a/stdlib/distutils/fancy_getopt.pyi +++ b/stdlib/distutils/fancy_getopt.pyi @@ -1,4 +1,3 @@ -from _typeshed import Incomplete from collections.abc import Iterable, Mapping from typing import Any, overload from typing_extensions import TypeAlias From 4365e68527b50c151294d113c502f6eacc41dcca Mon Sep 17 00:00:00 2001 From: Avasam Date: Sun, 19 Feb 2023 14:01:17 -0500 Subject: [PATCH 07/10] use packaging.requirements to get mypy version --- tests/stubtest_stdlib.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/stubtest_stdlib.py b/tests/stubtest_stdlib.py index 77c765de2a56..8e2de69dc54d 100755 --- a/tests/stubtest_stdlib.py +++ b/tests/stubtest_stdlib.py @@ -13,7 +13,7 @@ import tempfile from pathlib import Path -from pkg_resources import parse_requirements +from packaging.requirements import Requirement from utils import make_venv, print_error @@ -31,9 +31,16 @@ def run_stubtest(typeshed_dir: Path) -> int: print_error("fail") raise + # Install the same mypy version as in "requirements-tests.txt" with Path("requirements-tests.txt").open() as requirements_txt: - requirements = [str(requirement) for requirement in parse_requirements(requirements_txt) if requirement.key == "mypy"] - pip_cmd = [pip_exe, "install"] + requirements + mypy_requirement = next( + ( + Requirement(requirement.split("#")[0]) + for requirement in requirements_txt.readlines() + if requirement.startswith("mypy") + ) + ) + pip_cmd = [pip_exe, "install", str(mypy_requirement)] try: subprocess.run(pip_cmd, check=True, capture_output=True) except subprocess.CalledProcessError as e: From 1943c77c24b4b0665278777db5c67c847f7815eb Mon Sep 17 00:00:00 2001 From: Avasam Date: Sun, 19 Feb 2023 14:03:05 -0500 Subject: [PATCH 08/10] Fixes post PR comments --- stdlib/distutils/core.pyi | 1 + stdlib/distutils/cygwinccompiler.pyi | 3 ++- stdlib/distutils/dist.pyi | 3 ++- stdlib/distutils/fancy_getopt.pyi | 5 +++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/stdlib/distutils/core.pyi b/stdlib/distutils/core.pyi index 7bef94932beb..7b0bdd1b35bd 100644 --- a/stdlib/distutils/core.pyi +++ b/stdlib/distutils/core.pyi @@ -1,3 +1,4 @@ +from _typeshed import StrOrBytesPath from collections.abc import Mapping from distutils.cmd import Command as Command from distutils.dist import Distribution as Distribution diff --git a/stdlib/distutils/cygwinccompiler.pyi b/stdlib/distutils/cygwinccompiler.pyi index b45fdbbceae6..a990c3e28f36 100644 --- a/stdlib/distutils/cygwinccompiler.pyi +++ b/stdlib/distutils/cygwinccompiler.pyi @@ -1,5 +1,6 @@ from distutils.unixccompiler import UnixCCompiler from distutils.version import LooseVersion +from re import Pattern from typing_extensions import Literal def get_msvcr() -> list[str] | None: ... @@ -13,7 +14,7 @@ CONFIG_H_UNCERTAIN: str def check_config_h() -> tuple[Literal["ok", "not ok", "uncertain"], str]: ... -RE_VERSION: re.Pattern[bytes] +RE_VERSION: Pattern[bytes] def get_versions() -> tuple[LooseVersion | None, ...]: ... def is_cygwingcc() -> bool: ... diff --git a/stdlib/distutils/dist.pyi b/stdlib/distutils/dist.pyi index 0b9f80e11bdc..ceb61d63a637 100644 --- a/stdlib/distutils/dist.pyi +++ b/stdlib/distutils/dist.pyi @@ -1,9 +1,10 @@ from _typeshed import FileDescriptorOrPath, Incomplete, SupportsWrite from collections.abc import Iterable, Mapping from distutils.cmd import Command +from re import Pattern from typing import IO, Any -command_re: re.Pattern[str] +command_re: Pattern[str] class DistributionMetadata: def __init__(self, path: FileDescriptorOrPath | None = None) -> None: ... diff --git a/stdlib/distutils/fancy_getopt.pyi b/stdlib/distutils/fancy_getopt.pyi index fec289ac45a5..c15bb8a167dd 100644 --- a/stdlib/distutils/fancy_getopt.pyi +++ b/stdlib/distutils/fancy_getopt.pyi @@ -1,4 +1,5 @@ from collections.abc import Iterable, Mapping +from re import Pattern from typing import Any, overload from typing_extensions import TypeAlias @@ -6,8 +7,8 @@ _Option: TypeAlias = tuple[str, str | None, str] _GR: TypeAlias = tuple[list[str], OptionDummy] longopt_pat: str -longopt_re: re.Pattern[str] -neg_alias_re: re.Pattern[str] +longopt_re: Pattern[str] +neg_alias_re: Pattern[str] longopt_xlate: dict[int, int] class FancyGetopt: From d35e091b9038ca0bf5cb28877d5a00ea4bb488d1 Mon Sep 17 00:00:00 2001 From: Avasam Date: Sun, 19 Feb 2023 17:27:38 -0500 Subject: [PATCH 09/10] reuse get_mypy_req --- tests/stubtest_stdlib.py | 14 ++------------ tests/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/tests/stubtest_stdlib.py b/tests/stubtest_stdlib.py index 8e2de69dc54d..1e60d8d3a371 100755 --- a/tests/stubtest_stdlib.py +++ b/tests/stubtest_stdlib.py @@ -13,9 +13,7 @@ import tempfile from pathlib import Path -from packaging.requirements import Requirement - -from utils import make_venv, print_error +from utils import get_mypy_req, make_venv, print_error def run_stubtest(typeshed_dir: Path) -> int: @@ -32,15 +30,7 @@ def run_stubtest(typeshed_dir: Path) -> int: raise # Install the same mypy version as in "requirements-tests.txt" - with Path("requirements-tests.txt").open() as requirements_txt: - mypy_requirement = next( - ( - Requirement(requirement.split("#")[0]) - for requirement in requirements_txt.readlines() - if requirement.startswith("mypy") - ) - ) - pip_cmd = [pip_exe, "install", str(mypy_requirement)] + pip_cmd = [pip_exe, "install", get_mypy_req()] try: subprocess.run(pip_cmd, check=True, capture_output=True) except subprocess.CalledProcessError as e: diff --git a/tests/utils.py b/tests/utils.py index ab5011eddd90..ab1deb594161 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -81,8 +81,8 @@ def make_venv(venv_dir: Path) -> VenvInfo: @cache def get_mypy_req() -> str: - with open("requirements-tests.txt", encoding="UTF-8") as f: - return next(line.strip() for line in f if "mypy" in line) + with open("requirements-tests.txt", encoding="UTF-8") as requirements_file: + return next(strip_comments(line) for line in requirements_file if "mypy" in line) # ==================================================================== From 453848e6dc11c2d498ec118e5c5e3ca2609a4d4e Mon Sep 17 00:00:00 2001 From: Avasam Date: Tue, 21 Feb 2023 01:47:40 -0500 Subject: [PATCH 10/10] Apply PR review suggestions --- tests/stubtest_stdlib.py | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/tests/stubtest_stdlib.py b/tests/stubtest_stdlib.py index 1e60d8d3a371..75e1238fdba1 100755 --- a/tests/stubtest_stdlib.py +++ b/tests/stubtest_stdlib.py @@ -30,20 +30,10 @@ def run_stubtest(typeshed_dir: Path) -> int: raise # Install the same mypy version as in "requirements-tests.txt" - pip_cmd = [pip_exe, "install", get_mypy_req()] - try: - subprocess.run(pip_cmd, check=True, capture_output=True) - except subprocess.CalledProcessError as e: - print_command_failure("Failed to install mypy", e) - return e.returncode + subprocess.run([pip_exe, "install", get_mypy_req()], check=True) # Uninstall setuptools from the venv so we can test stdlib's distutils - pip_cmd = [pip_exe, "uninstall", "-y", "setuptools"] - try: - subprocess.run(pip_cmd, check=True, capture_output=True) - except subprocess.CalledProcessError as e: - print_command_failure("Failed to uninstall setuptools", e) - return e.returncode + subprocess.run([pip_exe, "uninstall", "-y", "setuptools"], check=True) cmd = [ python_exe, @@ -84,18 +74,5 @@ def run_stubtest(typeshed_dir: Path) -> int: return 0 -def print_command_failure(message: str, e: subprocess.CalledProcessError) -> None: - print_error("fail") - print(file=sys.stderr) - print(message, file=sys.stderr) - print_command_output(e) - - -def print_command_output(e: subprocess.CalledProcessError | subprocess.CompletedProcess[bytes]) -> None: - print(e.stdout.decode(), end="", file=sys.stderr) - print(e.stderr.decode(), end="", file=sys.stderr) - print(file=sys.stderr) - - if __name__ == "__main__": sys.exit(run_stubtest(typeshed_dir=Path(".")))