Fix stdlib/disutils testing (#9734)

.github/workflows/daily.yml

@@ -48,7 +48,7 @@ jobs:
           cache: pip
           cache-dependency-path: requirements-tests.txt
       - name: Install dependencies
-        run: pip install $(grep mypy== requirements-tests.txt)
+        run: pip install -r requirements-tests.txt
       - name: Run stubtest
         run: python tests/stubtest_stdlib.py
 

.github/workflows/stubtest_stdlib.yml

@@ -44,6 +44,6 @@ jobs:
           cache: pip
           cache-dependency-path: requirements-tests.txt
       - name: Install dependencies
-        run: pip install $(grep mypy== requirements-tests.txt)
+        run: pip install -r requirements-tests.txt
       - name: Run stubtest
         run: python tests/stubtest_stdlib.py

pyrightconfig.stricter.json

@@ -9,6 +9,7 @@
         // test cases use a custom pyrightconfig file
         "stubs/**/@tests/test_cases",
         "stdlib/distutils/command",
+        "stdlib/distutils/dist.pyi",
         "stdlib/lib2to3/refactor.pyi",
         "stdlib/_tkinter.pyi",
         "stdlib/tkinter/__init__.pyi",

stdlib/distutils/cmd.pyi

@@ -1,3 +1,4 @@
+from _typeshed import Incomplete
 from abc import abstractmethod
 from collections.abc import Callable, Iterable
 from distutils.dist import Distribution
@@ -60,3 +61,5 @@ class Command:
         skip_msg: str | None = None,
         level: Any = 1,
     ) -> None: ...  # level is not used
+    def ensure_finalized(self) -> None: ...
+    def dump_options(self, header: Incomplete | None = None, indent: str = "") -> None: ...

stdlib/distutils/core.pyi

@@ -1,9 +1,17 @@
+from _typeshed import StrOrBytesPath
 from collections.abc import Mapping
 from distutils.cmd import Command as Command
 from distutils.dist import Distribution as Distribution
 from distutils.extension import Extension as Extension
 from typing import Any
 
+USAGE: str
+
+def gen_usage(script_name: StrOrBytesPath) -> str: ...
+
+setup_keywords: tuple[str, ...]
+extension_keywords: tuple[str, ...]
+
 def setup(
     *,
     name: str = ...,

stdlib/distutils/cygwinccompiler.pyi

@@ -1,4 +1,20 @@
 from distutils.unixccompiler import UnixCCompiler
+from distutils.version import LooseVersion
+from re import Pattern
+from typing_extensions import Literal
+
+def get_msvcr() -> list[str] | None: ...
 
 class CygwinCCompiler(UnixCCompiler): ...
 class Mingw32CCompiler(CygwinCCompiler): ...
+
+CONFIG_H_OK: str
+CONFIG_H_NOTOK: str
+CONFIG_H_UNCERTAIN: str
+
+def check_config_h() -> tuple[Literal["ok", "not ok", "uncertain"], str]: ...
+
+RE_VERSION: Pattern[bytes]
+
+def get_versions() -> tuple[LooseVersion | None, ...]: ...
+def is_cygwingcc() -> bool: ...

stdlib/distutils/dist.pyi

@@ -1,8 +1,11 @@
-from _typeshed import FileDescriptorOrPath, SupportsWrite
+from _typeshed import FileDescriptorOrPath, Incomplete, SupportsWrite
 from collections.abc import Iterable, Mapping
 from distutils.cmd import Command
+from re import Pattern
 from typing import IO, Any
 
+command_re: Pattern[str]
+
 class DistributionMetadata:
     def __init__(self, path: FileDescriptorOrPath | None = None) -> None: ...
     name: str | None
@@ -57,3 +60,57 @@ class Distribution:
     def get_option_dict(self, command: str) -> dict[str, tuple[str, str]]: ...
     def parse_config_files(self, filenames: Iterable[str] | None = None) -> None: ...
     def get_command_obj(self, command: str, create: bool = ...) -> Command | None: ...
+    global_options: Incomplete
+    common_usage: str
+    display_options: Incomplete
+    display_option_names: Incomplete
+    negative_opt: Incomplete
+    verbose: int
+    dry_run: int
+    help: int
+    command_packages: Incomplete
+    script_name: Incomplete
+    script_args: Incomplete
+    command_options: Incomplete
+    dist_files: Incomplete
+    packages: Incomplete
+    package_data: Incomplete
+    package_dir: Incomplete
+    py_modules: Incomplete
+    libraries: Incomplete
+    headers: Incomplete
+    ext_modules: Incomplete
+    ext_package: Incomplete
+    include_dirs: Incomplete
+    extra_path: Incomplete
+    scripts: Incomplete
+    data_files: Incomplete
+    password: str
+    command_obj: Incomplete
+    have_run: Incomplete
+    want_user_cfg: bool
+    def dump_option_dicts(
+        self, header: Incomplete | None = ..., commands: Incomplete | None = ..., indent: str = ...
+    ) -> None: ...
+    def find_config_files(self): ...
+    commands: Incomplete
+    def parse_command_line(self): ...
+    def finalize_options(self) -> None: ...
+    def handle_display_options(self, option_order): ...
+    def print_command_list(self, commands, header, max_length) -> None: ...
+    def print_commands(self) -> None: ...
+    def get_command_list(self): ...
+    def get_command_packages(self): ...
+    def get_command_class(self, command): ...
+    def reinitialize_command(self, command, reinit_subcommands: int = ...): ...
+    def announce(self, msg, level=...) -> None: ...
+    def run_commands(self) -> None: ...
+    def run_command(self, command) -> None: ...
+    def has_pure_modules(self): ...
+    def has_ext_modules(self): ...
+    def has_c_libraries(self): ...
+    def has_modules(self): ...
+    def has_headers(self): ...
+    def has_scripts(self): ...
+    def has_data_files(self): ...
+    def is_pure(self): ...

stdlib/distutils/fancy_getopt.pyi

@@ -1,14 +1,15 @@
 from collections.abc import Iterable, Mapping
+from re import Pattern
 from typing import Any, overload
 from typing_extensions import TypeAlias
 
 _Option: TypeAlias = tuple[str, str | None, str]
 _GR: TypeAlias = tuple[list[str], OptionDummy]
 
-def fancy_getopt(
-    options: list[_Option], negative_opt: Mapping[_Option, _Option], object: Any, args: list[str] | None
-) -> list[str] | _GR: ...
-def wrap_text(text: str, width: int) -> list[str]: ...
+longopt_pat: str
+longopt_re: Pattern[str]
+neg_alias_re: Pattern[str]
+longopt_xlate: dict[int, int]
 
 class FancyGetopt:
     def __init__(self, option_table: list[_Option] | None = None) -> None: ...
@@ -20,5 +21,14 @@ class FancyGetopt:
     def get_option_order(self) -> list[tuple[str, str]]: ...
     def generate_help(self, header: str | None = None) -> list[str]: ...
 
+def fancy_getopt(
+    options: list[_Option], negative_opt: Mapping[_Option, _Option], object: Any, args: list[str] | None
+) -> list[str] | _GR: ...
+
+WS_TRANS: dict[int, str]
+
+def wrap_text(text: str, width: int) -> list[str]: ...
+def translate_longopt(opt: str) -> str: ...
+
 class OptionDummy:
     def __init__(self, options: Iterable[str] = ...) -> None: ...

stdlib/distutils/sysconfig.pyi

@@ -1,13 +1,22 @@
+import sys
 from collections.abc import Mapping
 from distutils.ccompiler import CCompiler
 
 PREFIX: str
 EXEC_PREFIX: str
+BASE_PREFIX: str
+BASE_EXEC_PREFIX: str
+project_base: str
+python_build: bool
 
+def expand_makefile_vars(s: str, vars: Mapping[str, str]) -> str: ...
 def get_config_var(name: str) -> int | str | None: ...
 def get_config_vars(*args: str) -> Mapping[str, int | str]: ...
 def get_config_h_filename() -> str: ...
 def get_makefile_filename() -> str: ...
 def get_python_inc(plat_specific: bool = ..., prefix: str | None = None) -> str: ...
 def get_python_lib(plat_specific: bool = ..., standard_lib: bool = ..., prefix: str | None = None) -> str: ...
 def customize_compiler(compiler: CCompiler) -> None: ...
+
+if sys.version_info < (3, 10):
+    def get_python_version() -> str: ...

stdlib/distutils/util.pyi

@@ -1,8 +1,12 @@
+import sys
 from _typeshed import StrPath, Unused
 from collections.abc import Callable, Container, Iterable, Mapping
 from typing import Any
 from typing_extensions import Literal
 
+if sys.version_info >= (3, 8):
+    def get_host_platform() -> str: ...
+
 def get_platform() -> str: ...
 def convert_path(pathname: str) -> str: ...
 def change_root(new_root: str, pathname: str) -> str: ...

tests/stubtest_allowlists/py310.txt

@@ -86,7 +86,6 @@ _collections_abc.MappingView.__class_getitem__
 _csv.Reader
 _csv.Writer
 bdb.Breakpoint.clearBreakpoints
-distutils.util.get_host_platform
 inspect.Signature.from_builtin  # Removed in 3.11, can add if someone needs this
 inspect.Signature.from_function  # Removed in 3.11, can add if someone needs this
 multiprocessing.managers.SharedMemoryServer.create
@@ -169,7 +168,3 @@ ast.ImportFrom.level  # None on the class, but never None on instances
 
 # White lies around defaults
 dataclasses.KW_ONLY
-
-# stubtest confuses stdlib distutils with setuptools-bundled distutils (#8410),
-# and the whole directory is going to be removed in 3.12 anyway
-distutils\..*

tests/stubtest_allowlists/py311.txt

@@ -139,7 +139,3 @@ typing._TypedDict.values
 
 # White lies around defaults
 dataclasses.KW_ONLY
-
-# stubtest confuses stdlib distutils with setuptools-bundled distutils (#8410),
-# and the whole directory is going to be removed in 3.12 anyway
-distutils\..*

tests/stubtest_allowlists/py37.txt

@@ -75,16 +75,6 @@ tkinter.Tk.__init__
 # Exists at runtime, but missing from stubs
 contextvars.ContextVar.__class_getitem__
 datetime.datetime_CAPI
-distutils.sysconfig.expand_makefile_vars
-distutils.sysconfig.get_python_version
-distutils.cygwinccompiler.RE_VERSION
-distutils.dist.command_re
-distutils.fancy_getopt.longopt_re
-distutils.fancy_getopt.neg_alias_re
-distutils.core.USAGE
-distutils.core.extension_keywords
-distutils.core.gen_usage
-distutils.core.setup_keywords
 dummy_threading.Lock
 dummy_threading.RLock
 html.parser.HTMLParser.unescape
@@ -162,54 +152,3 @@ types.GetSetDescriptorType.__get__
 types.MemberDescriptorType.__get__
 types.MethodDescriptorType.__get__
 types.WrapperDescriptorType.__get__
-
-# Missing from distutils (deprecated, to be removed in 3.12)
-distutils.core.USAGE
-distutils.core.extension_keywords
-distutils.core.gen_usage
-distutils.core.setup_keywords
-distutils.core.Command.dump_options
-distutils.core.Command.ensure_finalized
-distutils.core.Distribution.announce
-distutils.core.Distribution.common_usage
-distutils.core.Distribution.display_option_names
-distutils.core.Distribution.display_options
-distutils.core.Distribution.dump_option_dicts
-distutils.core.Distribution.find_config_files
-distutils.core.Distribution.get_command_packages
-distutils.core.Distribution.global_options
-distutils.core.Distribution.has_c_libraries
-distutils.core.Distribution.has_data_files
-distutils.core.Distribution.has_ext_modules
-distutils.core.Distribution.has_headers
-distutils.core.Distribution.has_modules
-distutils.core.Distribution.has_pure_modules
-distutils.core.Distribution.has_scripts
-distutils.core.Distribution.is_pure
-distutils.core.Distribution.negative_opt
-distutils.core.Distribution.parse_command_line
-distutils.core.Distribution.print_command_list
-distutils.core.Distribution.reinitialize_command
-distutils.core.Distribution.run_commands
-distutils.cygwinccompiler.is_cygwingcc
-distutils.dist.Distribution.announce
-distutils.dist.Distribution.common_usage
-distutils.dist.Distribution.display_option_names
-distutils.dist.Distribution.display_options
-distutils.dist.Distribution.dump_option_dicts
-distutils.dist.Distribution.find_config_files
-distutils.dist.Distribution.get_command_packages
-distutils.dist.Distribution.global_options
-distutils.dist.Distribution.has_c_libraries
-distutils.dist.Distribution.has_data_files
-distutils.dist.Distribution.has_ext_modules
-distutils.dist.Distribution.has_headers
-distutils.dist.Distribution.has_modules
-distutils.dist.Distribution.has_pure_modules
-distutils.dist.Distribution.has_scripts
-distutils.dist.Distribution.is_pure
-distutils.dist.Distribution.negative_opt
-distutils.dist.Distribution.parse_command_line
-distutils.dist.Distribution.print_command_list
-distutils.dist.Distribution.reinitialize_command
-distutils.dist.Distribution.run_commands

tests/stubtest_allowlists/py38.txt

@@ -35,10 +35,6 @@ collections.KeysView.__reversed__
 collections.ValuesView.__reversed__
 collections.Mapping.__reversed__  # Set to None at runtime for a better error message
 distutils.command.bdist_wininst  # see #6523
-distutils.core.USAGE
-distutils.core.extension_keywords
-distutils.core.gen_usage
-distutils.core.setup_keywords
 dummy_threading.Condition.acquire
 dummy_threading.Condition.release
 dummy_threading.Event.isSet
@@ -97,13 +93,6 @@ tkinter.Tk.__init__
 # Exists at runtime, but missing from stubs
 contextvars.ContextVar.__class_getitem__
 datetime.datetime_CAPI
-distutils.sysconfig.expand_makefile_vars
-distutils.sysconfig.get_python_version
-distutils.util.get_host_platform
-distutils.cygwinccompiler.RE_VERSION
-distutils.dist.command_re
-distutils.fancy_getopt.longopt_re
-distutils.fancy_getopt.neg_alias_re
 dummy_threading.ExceptHookArgs
 dummy_threading.Lock
 dummy_threading.RLock
@@ -180,54 +169,3 @@ types.GetSetDescriptorType.__get__
 types.MemberDescriptorType.__get__
 types.MethodDescriptorType.__get__
 types.WrapperDescriptorType.__get__
-
-# Missing from distutils (deprecated, to be removed in 3.12)
-distutils.core.USAGE
-distutils.core.extension_keywords
-distutils.core.gen_usage
-distutils.core.setup_keywords
-distutils.core.Command.dump_options
-distutils.core.Command.ensure_finalized
-distutils.core.Distribution.announce
-distutils.core.Distribution.common_usage
-distutils.core.Distribution.display_option_names
-distutils.core.Distribution.display_options
-distutils.core.Distribution.dump_option_dicts
-distutils.core.Distribution.find_config_files
-distutils.core.Distribution.get_command_packages
-distutils.core.Distribution.global_options
-distutils.core.Distribution.has_c_libraries
-distutils.core.Distribution.has_data_files
-distutils.core.Distribution.has_ext_modules
-distutils.core.Distribution.has_headers
-distutils.core.Distribution.has_modules
-distutils.core.Distribution.has_pure_modules
-distutils.core.Distribution.has_scripts
-distutils.core.Distribution.is_pure
-distutils.core.Distribution.negative_opt
-distutils.core.Distribution.parse_command_line
-distutils.core.Distribution.print_command_list
-distutils.core.Distribution.reinitialize_command
-distutils.core.Distribution.run_commands
-distutils.cygwinccompiler.is_cygwingcc
-distutils.dist.Distribution.announce
-distutils.dist.Distribution.common_usage
-distutils.dist.Distribution.display_option_names
-distutils.dist.Distribution.display_options
-distutils.dist.Distribution.dump_option_dicts
-distutils.dist.Distribution.find_config_files
-distutils.dist.Distribution.get_command_packages
-distutils.dist.Distribution.global_options
-distutils.dist.Distribution.has_c_libraries
-distutils.dist.Distribution.has_data_files
-distutils.dist.Distribution.has_ext_modules
-distutils.dist.Distribution.has_headers
-distutils.dist.Distribution.has_modules
-distutils.dist.Distribution.has_pure_modules
-distutils.dist.Distribution.has_scripts
-distutils.dist.Distribution.is_pure
-distutils.dist.Distribution.negative_opt
-distutils.dist.Distribution.parse_command_line
-distutils.dist.Distribution.print_command_list
-distutils.dist.Distribution.reinitialize_command
-distutils.dist.Distribution.run_commands