Add option to include docstrings with stubgen

docs/source/stubgen.rst

@@ -163,6 +163,11 @@ Additional flags
     Instead, only export imported names that are not referenced in the module
     that contains the import.
 
+.. option:: --include-docstrings
+
+    Include docstrings in stubs. This will add docstrings to Python function and
+    classes stubs and to C extension function stubs.
+
 .. option:: --search-path PATH
 
     Specify module search directories, separated by colons (only used if

misc/test-stubgenc.sh

@@ -3,17 +3,33 @@
 set -e
 set -x
 
-cd "$(dirname $0)/.."
+cd "$(dirname "$0")/.."
 
 # Install dependencies, demo project and mypy
 python -m pip install -r test-requirements.txt
 python -m pip install ./test-data/pybind11_mypy_demo
 python -m pip install .
 
-# Remove expected stubs and generate new inplace
-STUBGEN_OUTPUT_FOLDER=./test-data/pybind11_mypy_demo/stubgen
-rm -rf $STUBGEN_OUTPUT_FOLDER/*
-stubgen -p pybind11_mypy_demo -o $STUBGEN_OUTPUT_FOLDER
+EXIT=0
 
-# Compare generated stubs to expected ones
-git diff --exit-code $STUBGEN_OUTPUT_FOLDER
+# performs the stubgenc test
+# first argument is the test result folder
+# everything else is passed to stubgen as its arguments
+function stubgenc_test() {
+    # Remove expected stubs and generate new inplace
+    STUBGEN_OUTPUT_FOLDER=./test-data/pybind11_mypy_demo/$1
+    rm -rf "${STUBGEN_OUTPUT_FOLDER:?}/*"
+    stubgen -o "$STUBGEN_OUTPUT_FOLDER" "${@:2}"
+
+    # Compare generated stubs to expected ones
+    if ! git diff --exit-code "$STUBGEN_OUTPUT_FOLDER";
+    then
+        EXIT=$?
+    fi
+}
+
+# create stubs without docstrings
+stubgenc_test stubgen -p pybind11_mypy_demo  
+# create stubs with docstrings
+stubgenc_test stubgen-include-docs -p pybind11_mypy_demo --include-docstrings 
+exit $EXIT

mypy/fastparse.py

@@ -988,6 +988,8 @@ def do_func_def(
             # FuncDef overrides set_line -- can't use self.set_line
             func_def.set_line(lineno, n.col_offset, end_line, end_column)
             retval = func_def
+        if self.options.include_docstrings:
+            func_def.docstring = ast3.get_docstring(n, clean=False)
         self.class_and_function_stack.pop()
         return retval
 
@@ -1097,6 +1099,8 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef:
         else:
             cdef.line = n.lineno
             cdef.deco_line = n.decorator_list[0].lineno if n.decorator_list else None
+        if self.options.include_docstrings:
+            cdef.docstring = ast3.get_docstring(n, clean=False)
         cdef.column = n.col_offset
         cdef.end_line = getattr(n, "end_lineno", None)
         cdef.end_column = getattr(n, "end_col_offset", None)

mypy/nodes.py

@@ -782,6 +782,7 @@ class FuncDef(FuncItem, SymbolNode, Statement):
         "abstract_status",
         "original_def",
         "deco_line",
+        "docstring",
     )
 
     # Note that all __init__ args must have default values
@@ -802,6 +803,7 @@ def __init__(
         self.original_def: None | FuncDef | Var | Decorator = None
         # Used for error reporting (to keep backwad compatibility with pre-3.8)
         self.deco_line: int | None = None
+        self.docstring: str | None = None
 
     @property
     def name(self) -> str:
@@ -1081,6 +1083,7 @@ class ClassDef(Statement):
         "analyzed",
         "has_incompatible_baseclass",
         "deco_line",
+        "docstring",
     )
 
     name: str  # Name of the class without module prefix
@@ -1122,6 +1125,7 @@ def __init__(
         self.has_incompatible_baseclass = False
         # Used for error reporting (to keep backwad compatibility with pre-3.8)
         self.deco_line: int | None = None
+        self.docstring: str | None = None
 
     def accept(self, visitor: StatementVisitor[T]) -> T:
         return visitor.visit_class_def(self)

mypy/options.py

@@ -251,6 +251,12 @@ def __init__(self) -> None:
         # mypy. (Like mypyc.)
         self.preserve_asts = False
 
+        # If True, function and class docstrings will be extracted and retained.
+        # This isn't exposed as a command line option
+        # because it is intended for software integrating with
+        # mypy. (Like stubgen.)
+        self.include_docstrings = False
+
         # Paths of user plugins
         self.plugins: list[str] = []
 

mypy/stubgen.py

@@ -207,6 +207,7 @@ def __init__(
         verbose: bool,
         quiet: bool,
         export_less: bool,
+        include_docstrings: bool,
     ) -> None:
         # See parse_options for descriptions of the flags.
         self.pyversion = pyversion
@@ -225,6 +226,7 @@ def __init__(
         self.verbose = verbose
         self.quiet = quiet
         self.export_less = export_less
+        self.include_docstrings = include_docstrings
 
 
 class StubSource:
@@ -572,6 +574,7 @@ def __init__(
         include_private: bool = False,
         analyzed: bool = False,
         export_less: bool = False,
+        include_docstrings: bool = False,
     ) -> None:
         # Best known value of __all__.
         self._all_ = _all_
@@ -586,6 +589,7 @@ def __init__(
         self._state = EMPTY
         self._toplevel_names: list[str] = []
         self._include_private = include_private
+        self._include_docstrings = include_docstrings
         self.import_tracker = ImportTracker()
         # Was the tree semantically analysed before?
         self.analyzed = analyzed
@@ -755,7 +759,12 @@ def visit_func_def(
             retfield = " -> " + retname
 
         self.add(", ".join(args))
-        self.add(f"){retfield}: ...\n")
+        self.add(f"){retfield}:")
+        if self._include_docstrings and o.docstring:
+            self.add(f'\n{self._indent}    """{o.docstring}"""\n')
+        else:
+            self.add(" ...\n")
+
         self._state = FUNC
 
     def is_none_expr(self, expr: Expression) -> bool:
@@ -927,8 +936,10 @@ def visit_class_def(self, o: ClassDef) -> None:
         if base_types:
             self.add(f"({', '.join(base_types)})")
         self.add(":\n")
-        n = len(self._output)
         self._indent += "    "
+        if self._include_docstrings and o.docstring:
+            self.add(f'{self._indent}"""{o.docstring}"""\n')
+        n = len(self._output)
         self._vars.append([])
         super().visit_class_def(o)
         self._indent = self._indent[:-4]
@@ -937,7 +948,8 @@ def visit_class_def(self, o: ClassDef) -> None:
         if len(self._output) == n:
             if self._state == EMPTY_CLASS and sep is not None:
                 self._output[sep] = ""
-            self._output[-1] = self._output[-1][:-1] + " ...\n"
+            if not (self._include_docstrings and o.docstring):
+                self._output[-1] = self._output[-1][:-1] + " ...\n"
             self._state = EMPTY_CLASS
         else:
             self._state = CLASS
@@ -1549,6 +1561,7 @@ def mypy_options(stubgen_options: Options) -> MypyOptions:
     options.python_version = stubgen_options.pyversion
     options.show_traceback = True
     options.transform_source = remove_misplaced_type_comments
+    options.include_docstrings = stubgen_options.include_docstrings
     return options
 
 
@@ -1605,6 +1618,7 @@ def generate_stub_from_ast(
     parse_only: bool = False,
     include_private: bool = False,
     export_less: bool = False,
+    include_docstrings: bool = False,
 ) -> None:
     """Use analysed (or just parsed) AST to generate type stub for single file.
 
@@ -1616,6 +1630,7 @@ def generate_stub_from_ast(
         include_private=include_private,
         analyzed=not parse_only,
         export_less=export_less,
+        include_docstrings=include_docstrings,
     )
     assert mod.ast is not None, "This function must be used only with analyzed modules"
     mod.ast.accept(gen)
@@ -1670,7 +1685,12 @@ def generate_stubs(options: Options) -> None:
         files.append(target)
         with generate_guarded(mod.module, target, options.ignore_errors, options.verbose):
             generate_stub_from_ast(
-                mod, target, options.parse_only, options.include_private, options.export_less
+                mod,
+                target,
+                options.parse_only,
+                options.include_private,
+                options.export_less,
+                include_docstrings=options.include_docstrings,
             )
 
     # Separately analyse C modules using different logic.
@@ -1682,7 +1702,13 @@ def generate_stubs(options: Options) -> None:
         target = os.path.join(options.output_dir, target)
         files.append(target)
         with generate_guarded(mod.module, target, options.ignore_errors, options.verbose):
-            generate_stub_for_c_module(mod.module, target, sigs=sigs, class_sigs=class_sigs)
+            generate_stub_for_c_module(
+                mod.module,
+                target,
+                sigs=sigs,
+                class_sigs=class_sigs,
+                include_docstrings=options.include_docstrings,
+            )
     num_modules = len(py_modules) + len(c_modules)
     if not options.quiet and num_modules > 0:
         print("Processed %d modules" % num_modules)
@@ -1737,6 +1763,11 @@ def parse_options(args: list[str]) -> Options:
             "don't implicitly export all names imported from other modules " "in the same package"
         ),
     )
+    parser.add_argument(
+        "--include-docstrings",
+        action="store_true",
+        help=("include existing docstrings with the stubs"),
+    )
     parser.add_argument("-v", "--verbose", action="store_true", help="show more verbose messages")
     parser.add_argument("-q", "--quiet", action="store_true", help="show fewer messages")
     parser.add_argument(
@@ -1817,6 +1848,7 @@ def parse_options(args: list[str]) -> Options:
         verbose=ns.verbose,
         quiet=ns.quiet,
         export_less=ns.export_less,
+        include_docstrings=ns.include_docstrings,
     )
 
 

mypy/stubgenc.py

@@ -45,6 +45,7 @@ def generate_stub_for_c_module(
     target: str,
     sigs: dict[str, str] | None = None,
     class_sigs: dict[str, str] | None = None,
+    include_docstrings: bool = False,
 ) -> None:
     """Generate stub for C module.
 
@@ -65,15 +66,30 @@ def generate_stub_for_c_module(
     items = sorted(module.__dict__.items(), key=lambda x: x[0])
     for name, obj in items:
         if is_c_function(obj):
-            generate_c_function_stub(module, name, obj, functions, imports=imports, sigs=sigs)
+            generate_c_function_stub(
+                module,
+                name,
+                obj,
+                functions,
+                imports=imports,
+                sigs=sigs,
+                include_docstrings=include_docstrings,
+            )
             done.add(name)
     types: list[str] = []
     for name, obj in items:
         if name.startswith("__") and name.endswith("__"):
             continue
         if is_c_type(obj):
             generate_c_type_stub(
-                module, name, obj, types, imports=imports, sigs=sigs, class_sigs=class_sigs
+                module,
+                name,
+                obj,
+                types,
+                imports=imports,
+                sigs=sigs,
+                class_sigs=class_sigs,
+                include_docstrings=include_docstrings,
             )
             done.add(name)
     variables = []
@@ -157,10 +173,11 @@ def generate_c_function_stub(
     sigs: dict[str, str] | None = None,
     class_name: str | None = None,
     class_sigs: dict[str, str] | None = None,
+    include_docstrings: bool = False,
 ) -> None:
     """Generate stub for a single function or method.
 
-    The result (always a single line) will be appended to 'output'.
+    The result will be appended to 'output'.
     If necessary, any required names will be added to 'imports'.
     The 'class_name' is used to find signature of __init__ or __new__ in
     'class_sigs'.
@@ -171,7 +188,7 @@ def generate_c_function_stub(
         class_sigs = {}
 
     ret_type = "None" if name == "__init__" and class_name else "Any"
-
+    docstr = None
     if (
         name in ("__new__", "__init__")
         and name not in sigs
@@ -237,13 +254,24 @@ def generate_c_function_stub(
 
             if is_overloaded:
                 output.append("@overload")
-            output.append(
-                "def {function}({args}) -> {ret}: ...".format(
-                    function=name,
-                    args=", ".join(sig),
-                    ret=strip_or_import(signature.ret_type, module, imports),
+            if include_docstrings and docstr:
+                output.append(
+                    "def {function}({args}) -> {ret}:".format(
+                        function=name,
+                        args=", ".join(sig),
+                        ret=strip_or_import(signature.ret_type, module, imports),
+                    )
+                )
+                docstr_indented = "\n    ".join(docstr.strip().split("\n"))
+                output.extend(f'    """{docstr_indented}"""'.split("\n"))
+            else:
+                output.append(
+                    "def {function}({args}) -> {ret}: ...".format(
+                        function=name,
+                        args=", ".join(sig),
+                        ret=strip_or_import(signature.ret_type, module, imports),
+                    )
                 )
-            )
 
 
 def strip_or_import(typ: str, module: ModuleType, imports: list[str]) -> str:
@@ -340,6 +368,7 @@ def generate_c_type_stub(
     imports: list[str],
     sigs: dict[str, str] | None = None,
     class_sigs: dict[str, str] | None = None,
+    include_docstrings: bool = False,
 ) -> None:
     """Generate stub for a single class using runtime introspection.
 
@@ -383,6 +412,7 @@ def generate_c_type_stub(
                     sigs=sigs,
                     class_name=class_name,
                     class_sigs=class_sigs,
+                    include_docstrings=include_docstrings,
                 )
         elif is_c_property(value):
             done.add(attr)
@@ -398,7 +428,14 @@ def generate_c_type_stub(
             )
         elif is_c_type(value):
             generate_c_type_stub(
-                module, attr, value, types, imports=imports, sigs=sigs, class_sigs=class_sigs
+                module,
+                attr,
+                value,
+                types,
+                imports=imports,
+                sigs=sigs,
+                class_sigs=class_sigs,
+                include_docstrings=include_docstrings,
             )
             done.add(attr)
 

test-data/pybind11_mypy_demo/src/main.cpp

@@ -119,7 +119,7 @@ void bind_basics(py::module& basics) {
   using namespace basics;
 
   // Functions
-  basics.def("answer", &answer);
+  basics.def("answer", &answer, "answer docstring"); // tests explicit docstrings
   basics.def("sum", &sum);
   basics.def("midpoint", &midpoint, py::arg("left"), py::arg("right"));
   basics.def("weighted_midpoint", weighted_midpoint, py::arg("left"), py::arg("right"), py::arg("alpha")=0.5);