From ab12518c3be4539f41b52adfe854c03fe3de3bbe Mon Sep 17 00:00:00 2001 From: jakkdl Date: Wed, 12 Apr 2023 14:44:37 +0200 Subject: [PATCH] --no-edit --- trio/_path.py | 90 +++++++++++++++++-- trio/_path.pyi | 1 - trio/tests/test_exports.py | 176 +++++++++++++++++++++++++++++++++++-- 3 files changed, 254 insertions(+), 13 deletions(-) delete mode 100644 trio/_path.pyi diff --git a/trio/_path.py b/trio/_path.py index ea8cf98c34..656460a230 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -1,12 +1,27 @@ -# type: ignore - -from functools import wraps, partial import os -import types import pathlib +import sys +import types +from functools import partial, wraps +from typing import TYPE_CHECKING, Awaitable, Callable, TypeVar import trio -from trio._util import async_wraps, Final +from trio._util import Final, async_wraps + +if sys.version_info <= (3, 9): + from typing_extensions import ParamSpec +else: + from typing import ParamSpec + +if TYPE_CHECKING: + ArgsT = ParamSpec("ArgsT") + RetT = TypeVar("RetT") + + # mypy complains about missing `self` argument + def _make_async( + func: Callable[ArgsT, RetT] # type: ignore[misc] + ) -> Callable[ArgsT, Awaitable[RetT]]: + ... # re-wrap return value from methods that return new instances of pathlib.Path @@ -182,6 +197,71 @@ async def open(self, *args, **kwargs): value = await trio.to_thread.run_sync(func) return trio.wrap_file(value) + if TYPE_CHECKING: + # the dunders listed in _forward_magic that aren't seen otherwise + __bytes__ = pathlib.Path.__bytes__ + __truediv__ = pathlib.Path.__truediv__ + __rtruediv__ = pathlib.Path.__rtruediv__ + + # Note: these are not parsed properly by jedi. + # It might be superior to just manually implement all the methods and get rid + # of all the magic wrapping stuff. + + # wrapped methods handled by __getattr__ + absolute = _make_async(pathlib.Path.absolute) + as_posix = _make_async(pathlib.Path.as_posix) + as_uri = _make_async(pathlib.Path.as_uri) + chmod = _make_async(pathlib.Path.chmod) + cwd = _make_async(pathlib.Path.cwd) + exists = _make_async(pathlib.Path.exists) + expanduser = _make_async(pathlib.Path.expanduser) + glob = _make_async(pathlib.Path.glob) + group = _make_async(pathlib.Path.group) + home = _make_async(pathlib.Path.home) + is_absolute = _make_async(pathlib.Path.is_absolute) + is_block_device = _make_async(pathlib.Path.is_block_device) + is_char_device = _make_async(pathlib.Path.is_char_device) + is_dir = _make_async(pathlib.Path.is_dir) + is_fifo = _make_async(pathlib.Path.is_fifo) + is_file = _make_async(pathlib.Path.is_file) + is_mount = _make_async(pathlib.Path.is_mount) + is_reserved = _make_async(pathlib.Path.is_reserved) + is_socket = _make_async(pathlib.Path.is_socket) + is_symlink = _make_async(pathlib.Path.is_symlink) + iterdir = _make_async(pathlib.Path.iterdir) + joinpath = _make_async(pathlib.Path.joinpath) + lchmod = _make_async(pathlib.Path.lchmod) + lstat = _make_async(pathlib.Path.lstat) + match = _make_async(pathlib.Path.match) + mkdir = _make_async(pathlib.Path.mkdir) + owner = _make_async(pathlib.Path.owner) + read_bytes = _make_async(pathlib.Path.read_bytes) + read_text = _make_async(pathlib.Path.read_text) + relative_to = _make_async(pathlib.Path.relative_to) + rename = _make_async(pathlib.Path.rename) + replace = _make_async(pathlib.Path.replace) + resolve = _make_async(pathlib.Path.resolve) + rglob = _make_async(pathlib.Path.rglob) + rmdir = _make_async(pathlib.Path.rmdir) + samefile = _make_async(pathlib.Path.samefile) + stat = _make_async(pathlib.Path.stat) + symlink_to = _make_async(pathlib.Path.symlink_to) + touch = _make_async(pathlib.Path.touch) + unlink = _make_async(pathlib.Path.unlink) + with_name = _make_async(pathlib.Path.with_name) + with_suffix = _make_async(pathlib.Path.with_suffix) + write_bytes = _make_async(pathlib.Path.write_bytes) + write_text = _make_async(pathlib.Path.write_text) + + if sys.version_info >= (3, 8): + link_to = _make_async(pathlib.Path.link_to) + if sys.version_info >= (3, 9): + is_relative_to = _make_async(pathlib.Path.is_relative_to) + with_stem = _make_async(pathlib.Path.with_stem) + readlink = _make_async(pathlib.Path.readlink) + if sys.version_info >= (3, 10): + hardlink_to = _make_async(pathlib.Path.hardlink_to) + Path.iterdir.__doc__ = """ Like :meth:`pathlib.Path.iterdir`, but async. diff --git a/trio/_path.pyi b/trio/_path.pyi deleted file mode 100644 index 85a8e1f960..0000000000 --- a/trio/_path.pyi +++ /dev/null @@ -1 +0,0 @@ -class Path: ... diff --git a/trio/tests/test_exports.py b/trio/tests/test_exports.py index 026d6f5efa..e1fc964c5c 100644 --- a/trio/tests/test_exports.py +++ b/trio/tests/test_exports.py @@ -1,17 +1,22 @@ +import enum +import importlib +import inspect import re +import socket as stdlib_socket import sys -import importlib import types -import inspect -import enum +from pathlib import Path +from types import ModuleType +from typing import Any, Iterable import pytest import trio import trio.testing +from trio.tests.conftest import RUN_SLOW -from .. import _core -from .. import _util +from .. import _core, _util +from .._core.tests.tutil import slow def test_core_is_properly_reexported(): @@ -65,12 +70,12 @@ def public_modules(module): reason="skip static introspection tools on Python dev/alpha releases", ) @pytest.mark.parametrize("modname", PUBLIC_MODULE_NAMES) -@pytest.mark.parametrize("tool", ["pylint", "jedi"]) +@pytest.mark.parametrize("tool", ["pylint", "jedi", "mypy"]) @pytest.mark.filterwarnings( # https://github.com/pypa/setuptools/issues/3274 "ignore:module 'sre_constants' is deprecated:DeprecationWarning", ) -def test_static_tool_sees_all_symbols(tool, modname): +def test_static_tool_sees_all_symbols(tool, modname, tmpdir): module = importlib.import_module(modname) def no_underscores(symbols): @@ -96,6 +101,28 @@ def no_underscores(symbols): script = jedi.Script(f"import {modname}; {modname}.") completions = script.complete() static_names = no_underscores(c.name for c in completions) + elif tool == "mypy": + if not RUN_SLOW: + pytest.skip("use --run-slow to check against mypy") + + # create py.typed file + (Path(trio.__file__).parent / "py.typed").write_text("") + + # mypy behaves strangely when passed a huge semicolon-separated line with `-c` + # so we use a tmpfile + tmpfile = tmpdir / "check_mypy.py" + tmpfile.write_text( + f"import {modname}\n" + + "".join(f"{modname}.{name}\n" for name in runtime_names), + encoding="utf8", + ) + from mypy.api import run + + res = run(["--config-file=", "--follow-imports=silent", str(tmpfile)]) + + # check that there were no errors (exit code 0), otherwise print the errors + assert res[2] == 0, res[0] + return else: # pragma: no cover assert False @@ -114,6 +141,141 @@ def no_underscores(symbols): assert False +# this could be sped up by only invoking mypy once per module, or even once for all +# modules, instead of once per class. +@slow +# see commend on test_static_tool_sees_all_symbols +@pytest.mark.redistributors_should_skip +# pylint/jedi often have trouble with alpha releases, where Python's internals +# are in flux, grammar may not have settled down, etc. +@pytest.mark.skipif( + sys.version_info.releaselevel == "alpha", + reason="skip static introspection tools on Python dev/alpha releases", +) +@pytest.mark.parametrize("module_name", PUBLIC_MODULE_NAMES) +@pytest.mark.parametrize("tool", ["jedi", "mypy"]) +def test_static_tool_sees_class_members(tool, module_name, tmpdir) -> None: + module = PUBLIC_MODULES[PUBLIC_MODULE_NAMES.index(module_name)] + + # ignore hidden, but not dunder, symbols + def no_hidden(symbols): + return { + symbol + for symbol in symbols + if (not symbol.startswith("_")) or symbol.startswith("__") + } + + if tool == "mypy": + # create py.typed file + (Path(trio.__file__).parent / "py.typed").write_text("") + + errors: dict[str, Any] = {} + if module_name == "trio.tests": + return + for class_name, class_ in module.__dict__.items(): + if not isinstance(class_, type): + continue + if module_name == "trio.socket" and class_name in dir(stdlib_socket): + continue + # Deprecated classes are exported with a leading underscore + if class_name.startswith("_"): # pragma: no cover + continue + + # dir() and inspect.getmembers doesn't display properties from the metaclass + # also ignore some dunder methods that tend to differ but are of no consequence + ignore_names = set(dir(type(class_))) | { + "__annotations__", + "__attrs_attrs__", + "__attrs_own_setattr__", + "__class_getitem__", + "__getstate__", + "__match_args__", + "__order__", + "__orig_bases__", + "__parameters__", + "__setstate__", + "__slots__", + "__weakref__", + } + + # inspect.getmembers sees `name` and `value` in Enums, otherwise + # it behaves the same way as `dir` + # runtime_names = no_underscores(dir(class_)) + runtime_names = ( + no_hidden(x[0] for x in inspect.getmembers(class_)) - ignore_names + ) + + if tool == "jedi": + import jedi + + script = jedi.Script( + f"from {module_name} import {class_name}; {class_name}." + ) + completions = script.complete() + static_names = no_hidden(c.name for c in completions) - ignore_names + + missing = runtime_names - static_names + extra = static_names - runtime_names + if BaseException in class_.__mro__ and sys.version_info > (3, 11): + missing.remove("add_note") + + # TODO: why is this? Is it a problem? + if class_ == trio.StapledStream: + extra.remove("receive_stream") + extra.remove("send_stream") + + if missing | extra: + errors[f"{module_name}.{class_name}"] = { + "missing": missing, + "extra": extra, + } + elif tool == "mypy": + tmpfile = tmpdir / "check_mypy.py" + sorted_runtime_names = list(sorted(runtime_names)) + content = f"from {module_name} import {class_name}\n" + "".join( + f"{class_name}.{name}\n" for name in sorted_runtime_names + ) + tmpfile.write_text(content, encoding="utf8") + from mypy.api import run + + res = run( + [ + "--config-file=", + "--follow-imports=silent", + "--disable-error-code=operator", + str(tmpfile), + ] + ) + if res[2] != 0: + it = iter(res[0].split("\n")[:-2]) + for output_line in it: + kk = output_line.split(":") + + # -2 due to lines being 1-indexed and to skip the import line + symbol = ( + f"{module_name}.{class_name}." + + sorted_runtime_names[int(kk[1]) - 2] + ) + error = kk[2] + ":" + kk[3] + + # a bunch of symbols have this error, e.g. trio.lowlevel.Task.context + # but I don't think that's a problem - and if it is it would likely + # be picked up by "proper" mypy tests elsewhere + if "conflicts with class variable access" in error: + continue + + errors[symbol] = error + else: # pragma: no cover + assert False + + if errors: # pragma: no cover + from pprint import pprint + + print(f"\n{tool} can't see the following symbols in {module_name}:") + pprint(errors) + assert not errors + + def test_classes_are_final(): for module in PUBLIC_MODULES: for name, class_ in module.__dict__.items():