diff --git a/CHANGELOG.md b/CHANGELOG.md index f1d1e824..1535c48a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ + - pdoc now documents PyO3 or pybind11 submodules that are not picked up by Python's builtin pkgutil module. + ([#633](https://github.com/mitmproxy/pdoc/issues/633), @mhils) - Add support for `code-block` ReST directives ([#624](https://github.com/mitmproxy/pdoc/pull/624), @JCGoran) - If a variable's value meets certain entropy criteria and matches an environment variable value, diff --git a/pdoc/doc.py b/pdoc/doc.py index 2f00b1c8..352a0a33 100644 --- a/pdoc/doc.py +++ b/pdoc/doc.py @@ -28,7 +28,6 @@ import inspect import os from pathlib import Path -import pkgutil import re import sys import textwrap @@ -454,9 +453,6 @@ def own_members(self) -> list[Doc]: @cached_property def submodules(self) -> list[Module]: """A list of all (direct) submodules.""" - if not self.is_package: - return [] - include: Callable[[str], bool] mod_all = _safe_getattr(self.obj, "__all__", False) if mod_all is not False: @@ -471,9 +467,8 @@ def include(name: str) -> bool: # (think of OS-specific modules, e.g. _linux.py failing to import on Windows). return not name.startswith("_") - submodules = [] - for mod in pkgutil.iter_modules(self.obj.__path__, f"{self.fullname}."): - _, _, mod_name = mod.name.rpartition(".") + submodules: list[Module] = [] + for mod_name, mod in extract.iter_modules2(self.obj).items(): if not include(mod_name): continue try: diff --git a/pdoc/extract.py b/pdoc/extract.py index 2f32dd04..7d885035 100644 --- a/pdoc/extract.py +++ b/pdoc/extract.py @@ -229,22 +229,71 @@ def load_module(module: str) -> types.ModuleType: """ +def iter_modules2(module: types.ModuleType) -> dict[str, pkgutil.ModuleInfo]: + """ + Returns all direct child modules of a given module. + This function is similar to `pkgutil.iter_modules`, but + + 1. Respects a package's `__all__` attribute if specified. + If `__all__` is defined, submodules not listed in `__all__` are excluded. + 2. It will try to detect submodules that are not findable with iter_modules, + but are present in the module object. + """ + mod_all = getattr(module, "__all__", None) + + submodules = {} + + for submodule in pkgutil.iter_modules( + getattr(module, "__path__", []), f"{module.__name__}." + ): + name = submodule.name.rpartition(".")[2] + if mod_all is None or name in mod_all: + submodules[name] = submodule + + # 2023-12: PyO3 and pybind11 submodules are not detected by pkgutil + # This is a hacky workaround to register them. + members = dir(module) if mod_all is None else mod_all + for name in members: + if name in submodules or name == "__main__": + continue + member = getattr(module, name, None) + is_wild_child_module = ( + isinstance(member, types.ModuleType) + # the name is either just "bar", but can also be "foo.bar", + # see https://github.com/PyO3/pyo3/issues/759#issuecomment-1811992321 + and ( + member.__name__ == f"{module.__name__}.{name}" + or ( + member.__name__ == name + and sys.modules.get(member.__name__, None) is not member + ) + ) + ) + if is_wild_child_module: + # fixup the module name so that the rest of pdoc does not break + assert member + member.__name__ = f"{module.__name__}.{name}" + sys.modules[f"{module.__name__}.{name}"] = member + submodules[name] = pkgutil.ModuleInfo( + None, # type: ignore + name=f"{module.__name__}.{name}", + ispkg=True, + ) + + submodules.pop("__main__", None) # https://github.com/mitmproxy/pdoc/issues/438 + + return submodules + + def walk_packages2( modules: Iterable[pkgutil.ModuleInfo], ) -> Iterator[pkgutil.ModuleInfo]: """ For a given list of modules, recursively yield their names and all their submodules' names. - This function is similar to `pkgutil.walk_packages`, but respects a package's `__all__` attribute if specified. - If `__all__` is defined, submodules not listed in `__all__` are excluded. + This function is similar to `pkgutil.walk_packages`, but based on `iter_modules2`. """ - - # noinspection PyDefaultArgument - def seen(p, m={}): # pragma: no cover - if p in m: - return True - m[p] = True - + # the original walk_packages implementation has a recursion check for path, but that does not seem to be needed? for mod in modules: yield mod @@ -255,19 +304,8 @@ def seen(p, m={}): # pragma: no cover warnings.warn(f"Error loading {mod.name}:\n{traceback.format_exc()}") continue - mod_all = getattr(module, "__all__", None) - # don't traverse path items we've seen before - path = [p for p in (getattr(module, "__path__", None) or []) if not seen(p)] - - submodules = [] - for submodule in pkgutil.iter_modules(path, f"{mod.name}."): - name = submodule.name.rpartition(".")[2] - if name == "__main__": - continue # https://github.com/mitmproxy/pdoc/issues/438 - if mod_all is None or name in mod_all: - submodules.append(submodule) - - yield from walk_packages2(submodules) + submodules = iter_modules2(module) + yield from walk_packages2(submodules.values()) def module_mtime(modulename: str) -> float | None: diff --git a/pyproject.toml b/pyproject.toml index 5dcad820..3e2448c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ dev = [ "pytest-timeout", "hypothesis", "pygments >= 2.14.0", + "pdoc-pyo3-sample-library==1.0.11", ] [build-system] diff --git a/requirements-dev.txt b/requirements-dev.txt index a26f845e..92e1c7f3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,12 +2,10 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --all-extras --allow-unsafe --output-file=requirements-dev.txt --resolver=backtracking pyproject.toml +# pip-compile --all-extras --allow-unsafe --output-file=requirements-dev.txt pyproject.toml # attrs==23.1.0 - # via - # hypothesis - # pytest + # via hypothesis black==23.11.0 # via pdoc (pyproject.toml) cachetools==5.3.2 @@ -50,6 +48,8 @@ packaging==23.2 # tox pathspec==0.11.2 # via black +pdoc-pyo3-sample-library==1.0.11 + # via pdoc (pyproject.toml) platformdirs==4.0.0 # via # black @@ -79,14 +79,12 @@ sortedcontainers==2.4.0 tox==4.11.4 # via pdoc (pyproject.toml) types-docutils==0.20.0.3 - # via - # types-pygments - # types-setuptools + # via types-pygments types-pygments==2.17.0.0 # via pdoc (pyproject.toml) types-setuptools==69.0.0.0 # via types-pygments typing-extensions==4.8.0 # via mypy -virtualenv==20.24.7 +virtualenv==20.25.0 # via tox diff --git a/test/freeze-requirements.sh b/test/freeze-requirements.sh old mode 100644 new mode 100755 index 9690eedb..39e93920 --- a/test/freeze-requirements.sh +++ b/test/freeze-requirements.sh @@ -1,10 +1,5 @@ #!/usr/bin/env sh set -ex -cd -- "$(dirname -- "$0")" +cd -- "$(dirname -- "$0")/.." -rm -rf freeze-venv -python3.8 -m venv freeze-venv -freeze-venv/bin/python -m pip install -U pip -freeze-venv/bin/pip install -e ..[dev] -freeze-venv/bin/pip freeze --all --exclude-editable > ../requirements-dev.txt -rm -rf freeze-venv +pip-compile --all-extras --allow-unsafe --output-file=requirements-dev.txt pyproject.toml diff --git a/test/test_extract.py b/test/test_extract.py index 7da44761..800ec4ae 100644 --- a/test/test_extract.py +++ b/test/test_extract.py @@ -60,6 +60,14 @@ def test_walk_specs(): "test.mod_with_main.__main__", ] + assert walk_specs(["pdoc_pyo3_sample_library"]) == [ + "pdoc_pyo3_sample_library", + "pdoc_pyo3_sample_library.submodule", + "pdoc_pyo3_sample_library.submodule.subsubmodule", + "pdoc_pyo3_sample_library.explicit_submodule", + "pdoc_pyo3_sample_library.correct_name_submodule", + ] + def test_parse_spec(monkeypatch): p = sys.path diff --git a/test/test_snapshot.py b/test/test_snapshot.py index aeafa29d..23721d14 100755 --- a/test/test_snapshot.py +++ b/test/test_snapshot.py @@ -29,14 +29,14 @@ class Snapshot: def __init__( self, id: str, - filenames: list[str] | None = None, + specs: list[str] | None = None, render_options: dict | None = None, with_output_directory: bool = False, min_version: tuple[int, int] = (3, 7), warnings: list[str] | None = None, ): self.id = id - self.specs = filenames or [f"{id}.py"] + self.specs = specs or [f"{id}.py"] self.render_options = render_options or {} self.with_output_directory = with_output_directory self.min_version = min_version @@ -160,6 +160,7 @@ def outfile(self, format: str) -> Path: }, with_output_directory=True, ), + Snapshot("pyo3_sample_library", specs=["pdoc_pyo3_sample_library"]), Snapshot("top_level_reimports", ["top_level_reimports"]), Snapshot("type_checking_imports"), Snapshot("type_stub", min_version=(3, 10)), diff --git a/test/testdata/pyo3_sample_library.html b/test/testdata/pyo3_sample_library.html new file mode 100644 index 00000000..4b0107ce --- /dev/null +++ b/test/testdata/pyo3_sample_library.html @@ -0,0 +1,245 @@ + + + + + + + pdoc_pyo3_sample_library API documentation + + + + + + + + + +
+
+

+pdoc_pyo3_sample_library

+ +

This is a PyO3 demo library used to verify that submodules are handled properly.

+
+ + + + + + 
1from .pdoc_pyo3_sample_library import *
+2
+3__doc__ = pdoc_pyo3_sample_library.__doc__
+4if hasattr(pdoc_pyo3_sample_library, "__all__"):
+5    __all__ = pdoc_pyo3_sample_library.__all__
+
+ + +
+
+ + \ No newline at end of file diff --git a/test/testdata/pyo3_sample_library.txt b/test/testdata/pyo3_sample_library.txt new file mode 100644 index 00000000..5a429b70 --- /dev/null +++ b/test/testdata/pyo3_sample_library.txt @@ -0,0 +1 @@ + \ No newline at end of file