Detect Nonstandard Submodules

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 <!-- ✨ You do not need to add a pull request reference or an author, this will be added automatically by CI. ✨ -->
 
+ - pdoc now documents PyO3 or pybind11 submodules that are not picked up by Python's builtin pkgutil module.
+   ([#633](https://github.com/mitmproxy/pdoc/issues/633), @mhils)
  - Add support for `code-block` ReST directives
    ([#624](https://github.com/mitmproxy/pdoc/pull/624), @JCGoran)
  - If a variable's value meets certain entropy criteria and matches an environment variable value,

pdoc/doc.py

@@ -28,7 +28,6 @@
 import inspect
 import os
 from pathlib import Path
-import pkgutil
 import re
 import sys
 import textwrap
@@ -454,9 +453,6 @@ def own_members(self) -> list[Doc]:
     @cached_property
     def submodules(self) -> list[Module]:
         """A list of all (direct) submodules."""
-        if not self.is_package:
-            return []
-
         include: Callable[[str], bool]
         mod_all = _safe_getattr(self.obj, "__all__", False)
         if mod_all is not False:
@@ -471,9 +467,8 @@ def include(name: str) -> bool:
                 # (think of OS-specific modules, e.g. _linux.py failing to import on Windows).
                 return not name.startswith("_")
 
-        submodules = []
-        for mod in pkgutil.iter_modules(self.obj.__path__, f"{self.fullname}."):
-            _, _, mod_name = mod.name.rpartition(".")
+        submodules: list[Module] = []
+        for mod_name, mod in extract.iter_modules2(self.obj).items():
             if not include(mod_name):
                 continue
             try:

pdoc/extract.py

@@ -229,22 +229,71 @@ def load_module(module: str) -> types.ModuleType:
 """
 
 
+def iter_modules2(module: types.ModuleType) -> dict[str, pkgutil.ModuleInfo]:
+    """
+    Returns all direct child modules of a given module.
+    This function is similar to `pkgutil.iter_modules`, but
+
+      1. Respects a package's `__all__` attribute if specified.
+         If `__all__` is defined, submodules not listed in `__all__` are excluded.
+      2. It will try to detect submodules that are not findable with iter_modules,
+         but are present in the module object.
+    """
+    mod_all = getattr(module, "__all__", None)
+
+    submodules = {}
+
+    for submodule in pkgutil.iter_modules(
+        getattr(module, "__path__", []), f"{module.__name__}."
+    ):
+        name = submodule.name.rpartition(".")[2]
+        if mod_all is None or name in mod_all:
+            submodules[name] = submodule
+
+    # 2023-12: PyO3 and pybind11 submodules are not detected by pkgutil
+    # This is a hacky workaround to register them.
+    members = dir(module) if mod_all is None else mod_all
+    for name in members:
+        if name in submodules or name == "__main__":
+            continue
+        member = getattr(module, name, None)
+        is_wild_child_module = (
+            isinstance(member, types.ModuleType)
+            # the name is either just "bar", but can also be "foo.bar",
+            # see https://github.com/PyO3/pyo3/issues/759#issuecomment-1811992321
+            and (
+                member.__name__ == f"{module.__name__}.{name}"
+                or (
+                    member.__name__ == name
+                    and sys.modules.get(member.__name__, None) is not member
+                )
+            )
+        )
+        if is_wild_child_module:
+            # fixup the module name so that the rest of pdoc does not break
+            assert member
+            member.__name__ = f"{module.__name__}.{name}"
+            sys.modules[f"{module.__name__}.{name}"] = member
+            submodules[name] = pkgutil.ModuleInfo(
+                None,  # type: ignore
+                name=f"{module.__name__}.{name}",
+                ispkg=True,
+            )
+
+    submodules.pop("__main__", None)  # https://github.com/mitmproxy/pdoc/issues/438
+
+    return submodules
+
+
 def walk_packages2(
     modules: Iterable[pkgutil.ModuleInfo],
 ) -> Iterator[pkgutil.ModuleInfo]:
     """
     For a given list of modules, recursively yield their names and all their submodules' names.
 
-    This function is similar to `pkgutil.walk_packages`, but respects a package's `__all__` attribute if specified.
-    If `__all__` is defined, submodules not listed in `__all__` are excluded.
+    This function is similar to `pkgutil.walk_packages`, but based on `iter_modules2`.
     """
-
-    # noinspection PyDefaultArgument
-    def seen(p, m={}):  # pragma: no cover
-        if p in m:
-            return True
-        m[p] = True
-
+    # the original walk_packages implementation has a recursion check for path, but that does not seem to be needed?
     for mod in modules:
         yield mod
 
@@ -255,19 +304,8 @@ def seen(p, m={}):  # pragma: no cover
                 warnings.warn(f"Error loading {mod.name}:\n{traceback.format_exc()}")
                 continue
 
-            mod_all = getattr(module, "__all__", None)
-            # don't traverse path items we've seen before
-            path = [p for p in (getattr(module, "__path__", None) or []) if not seen(p)]
-
-            submodules = []
-            for submodule in pkgutil.iter_modules(path, f"{mod.name}."):
-                name = submodule.name.rpartition(".")[2]
-                if name == "__main__":
-                    continue  # https://github.com/mitmproxy/pdoc/issues/438
-                if mod_all is None or name in mod_all:
-                    submodules.append(submodule)
-
-            yield from walk_packages2(submodules)
+            submodules = iter_modules2(module)
+            yield from walk_packages2(submodules.values())
 
 
 def module_mtime(modulename: str) -> float | None:

pyproject.toml

@@ -52,6 +52,7 @@ dev = [
     "pytest-timeout",
     "hypothesis",
     "pygments >= 2.14.0",
+    "pdoc-pyo3-sample-library==1.0.11",
 ]
 
 [build-system]

requirements-dev.txt

@@ -2,12 +2,10 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --all-extras --allow-unsafe --output-file=requirements-dev.txt --resolver=backtracking pyproject.toml
+#    pip-compile --all-extras --allow-unsafe --output-file=requirements-dev.txt pyproject.toml
 #
 attrs==23.1.0
-    # via
-    #   hypothesis
-    #   pytest
+    # via hypothesis
 black==23.11.0
     # via pdoc (pyproject.toml)
 cachetools==5.3.2
@@ -50,6 +48,8 @@ packaging==23.2
     #   tox
 pathspec==0.11.2
     # via black
+pdoc-pyo3-sample-library==1.0.11
+    # via pdoc (pyproject.toml)
 platformdirs==4.0.0
     # via
     #   black
@@ -79,14 +79,12 @@ sortedcontainers==2.4.0
 tox==4.11.4
     # via pdoc (pyproject.toml)
 types-docutils==0.20.0.3
-    # via
-    #   types-pygments
-    #   types-setuptools
+    # via types-pygments
 types-pygments==2.17.0.0
     # via pdoc (pyproject.toml)
 types-setuptools==69.0.0.0
     # via types-pygments
 typing-extensions==4.8.0
     # via mypy
-virtualenv==20.24.7
+virtualenv==20.25.0
     # via tox

test/freeze-requirements.sh

@@ -1,10 +1,5 @@
 #!/usr/bin/env sh
 set -ex
-cd -- "$(dirname -- "$0")"
+cd -- "$(dirname -- "$0")/.."
 
-rm -rf freeze-venv
-python3.8 -m venv freeze-venv
-freeze-venv/bin/python -m pip install -U pip
-freeze-venv/bin/pip install -e ..[dev]
-freeze-venv/bin/pip freeze --all --exclude-editable > ../requirements-dev.txt
-rm -rf freeze-venv
+pip-compile --all-extras --allow-unsafe --output-file=requirements-dev.txt pyproject.toml

test/test_extract.py

@@ -60,6 +60,14 @@ def test_walk_specs():
         "test.mod_with_main.__main__",
     ]
 
+    assert walk_specs(["pdoc_pyo3_sample_library"]) == [
+        "pdoc_pyo3_sample_library",
+        "pdoc_pyo3_sample_library.submodule",
+        "pdoc_pyo3_sample_library.submodule.subsubmodule",
+        "pdoc_pyo3_sample_library.explicit_submodule",
+        "pdoc_pyo3_sample_library.correct_name_submodule",
+    ]
+
 
 def test_parse_spec(monkeypatch):
     p = sys.path

test/test_snapshot.py

@@ -29,14 +29,14 @@ class Snapshot:
     def __init__(
         self,
         id: str,
-        filenames: list[str] | None = None,
+        specs: list[str] | None = None,
         render_options: dict | None = None,
         with_output_directory: bool = False,
         min_version: tuple[int, int] = (3, 7),
         warnings: list[str] | None = None,
     ):
         self.id = id
-        self.specs = filenames or [f"{id}.py"]
+        self.specs = specs or [f"{id}.py"]
         self.render_options = render_options or {}
         self.with_output_directory = with_output_directory
         self.min_version = min_version
@@ -160,6 +160,7 @@ def outfile(self, format: str) -> Path:
         },
         with_output_directory=True,
     ),
+    Snapshot("pyo3_sample_library", specs=["pdoc_pyo3_sample_library"]),
     Snapshot("top_level_reimports", ["top_level_reimports"]),
     Snapshot("type_checking_imports"),
     Snapshot("type_stub", min_version=(3, 10)),

test/testdata/pyo3_sample_library.txt

@@ -0,0 +1 @@
+<module pdoc_pyo3_sample_library  # This is a PyO3 demo …>