From 8a78e68e7702d5b874daa21aeca58943eb9cf7d6 Mon Sep 17 00:00:00 2001 From: Marco Ricci Date: Fri, 6 Sep 2024 20:24:54 +0200 Subject: [PATCH 1/3] feat: Allow deselecting multiple or named items in Yields and Receives Issue-263: https://github.com/mkdocstrings/griffe/issues/263 --- src/_griffe/docstrings/google.py | 78 +++++++--- tests/test_docstrings/test_google.py | 218 +++++++++++++++++++++++++++ 2 files changed, 272 insertions(+), 24 deletions(-) diff --git a/src/_griffe/docstrings/google.py b/src/_griffe/docstrings/google.py index db3b9b37..85539910 100644 --- a/src/_griffe/docstrings/google.py +++ b/src/_griffe/docstrings/google.py @@ -521,22 +521,37 @@ def _read_yields_section( docstring: Docstring, *, offset: int, + returns_multiple_items: bool = True, + returns_named_value: bool = True, **options: Any, ) -> tuple[DocstringSectionYields | None, int]: yields = [] - block, new_offset = _read_block_items(docstring, offset=offset, **options) - for index, (line_number, yield_lines) in enumerate(block): - match = _RE_NAME_ANNOTATION_DESCRIPTION.match(yield_lines[0]) - if not match: - docstring_warning( - docstring, - line_number, - f"Failed to get name, annotation or description from '{yield_lines[0]}'", - ) - continue + if returns_multiple_items: + block, new_offset = _read_block_items(docstring, offset=offset, **options) + else: + one_block, new_offset = _read_block(docstring, offset=offset, **options) + block = [(new_offset, one_block.splitlines())] - name, annotation, description = match.groups() + for index, (line_number, yield_lines) in enumerate(block): + if returns_named_value: + match = _RE_NAME_ANNOTATION_DESCRIPTION.match(yield_lines[0]) + if not match: + docstring_warning( + docstring, + line_number, + f"Failed to get name, annotation or description from '{yield_lines[0]}'", + ) + continue + name, annotation, description = match.groups() + else: + name = None + if ":" in yield_lines[0]: + annotation, description = yield_lines[0].split(":", 1) + annotation = annotation.lstrip("(").rstrip(")") + else: + annotation = None + description = yield_lines[0] description = "\n".join([description.lstrip(), *yield_lines[1:]]).rstrip("\n") if annotation: @@ -554,7 +569,7 @@ def _read_yields_section( raise ValueError if isinstance(yield_item, ExprName): annotation = yield_item - elif yield_item.is_tuple: + elif yield_item.is_tuple and returns_multiple_items: annotation = yield_item.slice.elements[index] else: annotation = yield_item @@ -572,22 +587,37 @@ def _read_receives_section( docstring: Docstring, *, offset: int, + receives_multiple_items: bool = True, + receives_named_value: bool = True, **options: Any, ) -> tuple[DocstringSectionReceives | None, int]: receives = [] - block, new_offset = _read_block_items(docstring, offset=offset, **options) - for index, (line_number, receive_lines) in enumerate(block): - match = _RE_NAME_ANNOTATION_DESCRIPTION.match(receive_lines[0]) - if not match: - docstring_warning( - docstring, - line_number, - f"Failed to get name, annotation or description from '{receive_lines[0]}'", - ) - continue + if receives_multiple_items: + block, new_offset = _read_block_items(docstring, offset=offset, **options) + else: + one_block, new_offset = _read_block(docstring, offset=offset, **options) + block = [(new_offset, one_block.splitlines())] - name, annotation, description = match.groups() + for index, (line_number, receive_lines) in enumerate(block): + if receives_multiple_items: + match = _RE_NAME_ANNOTATION_DESCRIPTION.match(receive_lines[0]) + if not match: + docstring_warning( + docstring, + line_number, + f"Failed to get name, annotation or description from '{receive_lines[0]}'", + ) + continue + name, annotation, description = match.groups() + else: + name = None + if ":" in receive_lines[0]: + annotation, description = receive_lines[0].split(":", 1) + annotation = annotation.lstrip("(").rstrip(")") + else: + annotation = None + description = receive_lines[0] description = "\n".join([description.lstrip(), *receive_lines[1:]]).rstrip("\n") if annotation: @@ -601,7 +631,7 @@ def _read_receives_section( receives_item = annotation.slice.elements[1] if isinstance(receives_item, ExprName): annotation = receives_item - elif receives_item.is_tuple: + elif receives_item.is_tuple and receives_multiple_items: annotation = receives_item.slice.elements[index] else: annotation = receives_item diff --git a/tests/test_docstrings/test_google.py b/tests/test_docstrings/test_google.py index 4dae96c5..15037b22 100644 --- a/tests/test_docstrings/test_google.py +++ b/tests/test_docstrings/test_google.py @@ -11,8 +11,10 @@ Attribute, Class, Docstring, + DocstringReceive, DocstringReturn, DocstringSectionKind, + DocstringYield, ExprName, Function, Module, @@ -1407,6 +1409,148 @@ def test_parse_returns_multiple_items( assert annotated.description == expected_.description +@pytest.mark.parametrize( + ("returns_multiple_items", "return_annotation", "expected"), + [ + ( + False, + None, + [DocstringYield("", description="XXXXXXX\n YYYYYYY\nZZZZZZZ", annotation=None)], + ), + ( + False, + "Iterator[tuple[int, int]]", + [DocstringYield("", description="XXXXXXX\n YYYYYYY\nZZZZZZZ", annotation="tuple[int, int]")], + ), + ( + True, + None, + [ + DocstringYield("", description="XXXXXXX\nYYYYYYY", annotation=None), + DocstringYield("", description="ZZZZZZZ", annotation=None), + ], + ), + ( + True, + "Iterator[tuple[int,int]]", + [ + DocstringYield("", description="XXXXXXX\nYYYYYYY", annotation="int"), + DocstringYield("", description="ZZZZZZZ", annotation="int"), + ], + ), + ], +) +def test_parse_yields_multiple_items( + parse_google: ParserType, + returns_multiple_items: bool, + return_annotation: str, + expected: list[DocstringYield], +) -> None: + """Parse Returns section with and without multiple items. + + Parameters: + parse_google: Fixture parser. + returns_multiple_items: Whether the `Returns` and `Yields` sections have multiple items. + return_annotation: The return annotation of the function to parse. Usually an `Iterator`. + expected: The expected value of the parsed Yields section. + """ + parent = ( + Function("func", returns=parse_docstring_annotation(return_annotation, Docstring("d", parent=Function("f")))) + if return_annotation is not None + else None + ) + docstring = """ + Yields: + XXXXXXX + YYYYYYY + ZZZZZZZ + """ + sections, _ = parse_google( + docstring, + returns_multiple_items=returns_multiple_items, + parent=parent, + ) + + assert len(sections) == 1 + assert len(sections[0].value) == len(expected) + + for annotated, expected_ in zip(sections[0].value, expected): + assert annotated.name == expected_.name + assert str(annotated.annotation) == str(expected_.annotation) + assert annotated.description == expected_.description + + +@pytest.mark.parametrize( + ("receives_multiple_items", "return_annotation", "expected"), + [ + ( + False, + None, + [DocstringReceive("", description="XXXXXXX\n YYYYYYY\nZZZZZZZ", annotation=None)], + ), + ( + False, + "Generator[..., tuple[int, int], ...]", + [DocstringReceive("", description="XXXXXXX\n YYYYYYY\nZZZZZZZ", annotation="tuple[int, int]")], + ), + ( + True, + None, + [ + DocstringReceive("", description="XXXXXXX\nYYYYYYY", annotation=None), + DocstringReceive("", description="ZZZZZZZ", annotation=None), + ], + ), + ( + True, + "Generator[..., tuple[int, int], ...]", + [ + DocstringReceive("", description="XXXXXXX\nYYYYYYY", annotation="int"), + DocstringReceive("", description="ZZZZZZZ", annotation="int"), + ], + ), + ], +) +def test_parse_receives_multiple_items( + parse_google: ParserType, + receives_multiple_items: bool, + return_annotation: str, + expected: list[DocstringReceive], +) -> None: + """Parse Returns section with and without multiple items. + + Parameters: + parse_google: Fixture parser. + receives_multiple_items: Whether the `Receives` section has multiple items. + return_annotation: The return annotation of the function to parse. Usually a `Generator`. + expected: The expected value of the parsed Receives section. + """ + parent = ( + Function("func", returns=parse_docstring_annotation(return_annotation, Docstring("d", parent=Function("f")))) + if return_annotation is not None + else None + ) + docstring = """ + Receives: + XXXXXXX + YYYYYYY + ZZZZZZZ + """ + sections, _ = parse_google( + docstring, + receives_multiple_items=receives_multiple_items, + parent=parent, + ) + + assert len(sections) == 1 + assert len(sections[0].value) == len(expected) + + for annotated, expected_ in zip(sections[0].value, expected): + assert annotated.name == expected_.name + assert str(annotated.annotation) == str(expected_.annotation) + assert annotated.description == expected_.description + + def test_avoid_false_positive_sections(parse_google: ParserType) -> None: """Avoid false positive when parsing sections. @@ -1490,6 +1634,80 @@ def test_type_in_returns_without_parentheses(parse_google: ParserType) -> None: assert retval.description == "Description\non several lines." +def test_type_in_yields_without_parentheses(parse_google: ParserType) -> None: + """Assert we can parse the return type without parentheses. + + Parameters: + parse_google: Fixture parser. + """ + docstring = """ + Summary. + + Yields: + int: Description + on several lines. + """ + sections, warnings = parse_google(docstring, returns_named_value=False) + assert len(sections) == 2 + assert not warnings + retval = sections[1].value[0] + assert retval.name == "" + assert retval.annotation == "int" + assert retval.description == "Description\non several lines." + + docstring = """ + Summary. + + Yields: + Description + on several lines. + """ + sections, warnings = parse_google(docstring, returns_named_value=False) + assert len(sections) == 2 + assert len(warnings) == 1 + retval = sections[1].value[0] + assert retval.name == "" + assert retval.annotation is None + assert retval.description == "Description\non several lines." + + +def test_type_in_receives_without_parentheses(parse_google: ParserType) -> None: + """Assert we can parse the return type without parentheses. + + Parameters: + parse_google: Fixture parser. + """ + docstring = """ + Summary. + + Receives: + int: Description + on several lines. + """ + sections, warnings = parse_google(docstring, receives_named_value=False) + assert len(sections) == 2 + assert not warnings + retval = sections[1].value[0] + assert retval.name == "" + assert retval.annotation == "int" + assert retval.description == "Description\non several lines." + + docstring = """ + Summary. + + Receives: + Description + on several lines. + """ + sections, warnings = parse_google(docstring, receives_named_value=False) + assert len(sections) == 2 + assert len(warnings) == 1 + retval = sections[1].value[0] + assert retval.name == "" + assert retval.annotation is None + assert retval.description == "Description\non several lines." + + def test_reading_property_type_in_summary(parse_google: ParserType) -> None: """Assert we can parse the return type of properties in their summary. From 4695f04451966a15cd00e32e789d1d4e503de6ec Mon Sep 17 00:00:00 2001 From: Marco Ricci Date: Fri, 6 Sep 2024 21:51:33 +0200 Subject: [PATCH 2/3] refactor: Extract common functionality in Returns, Yields and Receives parsing The Returns, Yields and Receives section parsers only really differ in their fallbacks and the names of their configuration settings, not so much in their general parsing behavior and the expected formatting of the section contents. This commit is an attempt to factor out the following functionality: * Read the section contents as a single block, or as multiple blocks, depending on the `multiple` setting. * Parse each block's first line as a named parameter, or an unnamed parameter, depending on the `named` setting. * Unpack `Generator` and `Iterator` types in the return annotation. Optionally error out if the return annotation is not of these types. * Optionally destructure the return tuple if `multiple` is in effect. Issue-263: https://github.com/mkdocstrings/griffe/issues/263 --- src/_griffe/docstrings/google.py | 238 +++++++++++++++++-------------- 1 file changed, 130 insertions(+), 108 deletions(-) diff --git a/src/_griffe/docstrings/google.py b/src/_griffe/docstrings/google.py index 85539910..40680e22 100644 --- a/src/_griffe/docstrings/google.py +++ b/src/_griffe/docstrings/google.py @@ -442,6 +442,76 @@ def _read_warns_section( return DocstringSectionWarns(warns), new_offset +def _read_block_items_maybe( + docstring: Docstring, + *, + offset: int, + multiple: bool = True, + **options: Any, +) -> _ItemsBlock: + if multiple: + return _read_block_items(docstring, offset=offset, **options) + one_block, new_offset = _read_block(docstring, offset=offset, **options) + return [(new_offset, one_block.splitlines())], new_offset + + +def _get_name_annotation_description( + docstring: Docstring, + line_number: int, + lines: list[str], + *, + named: bool = True, +) -> tuple[str | None, Any, str]: + if named: + match = _RE_NAME_ANNOTATION_DESCRIPTION.match(lines[0]) + if not match: + docstring_warning( + docstring, + line_number, + f"Failed to get name, annotation or description from '{lines[0]}'", + ) + raise ValueError + name, annotation, description = match.groups() + else: + name = None + if ":" in lines[0]: + annotation, description = lines[0].split(":", 1) + annotation = annotation.lstrip("(").rstrip(")") + else: + annotation = None + description = lines[0] + description = "\n".join([description.lstrip(), *lines[1:]]).rstrip("\n") + return name, annotation, description + + +def _unpack_generators( + annotation: Any, + generator_pos: int, + *, + mandatory: bool = False, +) -> Any: + if annotation.is_generator: + return annotation.slice.elements[generator_pos] + if annotation.is_iterator: + return annotation.slice + if mandatory: + raise ValueError(f"must be a Generator: {annotation!r}") + return annotation + + +def _maybe_destructure_annotation( + annotation: Any, + index: int, + *, + multiple: bool = True, +) -> Any: + if isinstance(annotation, ExprName): + return annotation + if multiple and annotation.is_tuple: + return annotation.slice.elements[index] + return annotation + + def _read_returns_section( docstring: Docstring, *, @@ -452,32 +522,23 @@ def _read_returns_section( ) -> tuple[DocstringSectionReturns | None, int]: returns = [] - if returns_multiple_items: - block, new_offset = _read_block_items(docstring, offset=offset, **options) - else: - one_block, new_offset = _read_block(docstring, offset=offset, **options) - block = [(new_offset, one_block.splitlines())] + block, new_offset = _read_block_items_maybe( + docstring, + offset=offset, + multiple=returns_multiple_items, + **options, + ) for index, (line_number, return_lines) in enumerate(block): - if returns_named_value: - match = _RE_NAME_ANNOTATION_DESCRIPTION.match(return_lines[0]) - if not match: - docstring_warning( - docstring, - line_number, - f"Failed to get name, annotation or description from '{return_lines[0]}'", - ) - continue - name, annotation, description = match.groups() - else: - name = None - if ":" in return_lines[0]: - annotation, description = return_lines[0].split(":", 1) - annotation = annotation.lstrip("(").rstrip(")") - else: - annotation = None - description = return_lines[0] - description = "\n".join([description.lstrip(), *return_lines[1:]]).rstrip("\n") + try: + name, annotation, description = _get_name_annotation_description( + docstring, + line_number, + return_lines, + named=returns_named_value, + ) + except ValueError: + continue if annotation: # try to compile the annotation to transform it into an expression @@ -491,22 +552,11 @@ def _read_returns_section( annotation = docstring.parent.annotation # type: ignore[union-attr] else: raise ValueError - if len(block) > 1: - if annotation.is_tuple: - annotation = annotation.slice.elements[index] - else: - if annotation.is_iterator: - return_item = annotation.slice - elif annotation.is_generator: - return_item = annotation.slice.elements[2] - else: - raise ValueError - if isinstance(return_item, ExprName): - annotation = return_item - elif return_item.is_tuple: - annotation = return_item.slice.elements[index] - else: - annotation = return_item + annotation = _maybe_destructure_annotation( + _unpack_generators(annotation, 2), + index, + multiple=returns_multiple_items, + ) if annotation is None: returned_value = repr(name) if name else index + 1 @@ -527,32 +577,23 @@ def _read_yields_section( ) -> tuple[DocstringSectionYields | None, int]: yields = [] - if returns_multiple_items: - block, new_offset = _read_block_items(docstring, offset=offset, **options) - else: - one_block, new_offset = _read_block(docstring, offset=offset, **options) - block = [(new_offset, one_block.splitlines())] + block, new_offset = _read_block_items_maybe( + docstring, + offset=offset, + multiple=returns_multiple_items, + **options, + ) for index, (line_number, yield_lines) in enumerate(block): - if returns_named_value: - match = _RE_NAME_ANNOTATION_DESCRIPTION.match(yield_lines[0]) - if not match: - docstring_warning( - docstring, - line_number, - f"Failed to get name, annotation or description from '{yield_lines[0]}'", - ) - continue - name, annotation, description = match.groups() - else: - name = None - if ":" in yield_lines[0]: - annotation, description = yield_lines[0].split(":", 1) - annotation = annotation.lstrip("(").rstrip(")") - else: - annotation = None - description = yield_lines[0] - description = "\n".join([description.lstrip(), *yield_lines[1:]]).rstrip("\n") + try: + name, annotation, description = _get_name_annotation_description( + docstring, + line_number, + yield_lines, + named=returns_named_value, + ) + except ValueError: + continue if annotation: # try to compile the annotation to transform it into an expression @@ -561,18 +602,11 @@ def _read_yields_section( # try to retrieve the annotation from the docstring parent with suppress(AttributeError, IndexError, KeyError, ValueError): annotation = docstring.parent.annotation # type: ignore[union-attr] - if annotation.is_iterator: - yield_item = annotation.slice - elif annotation.is_generator: - yield_item = annotation.slice.elements[0] - else: - raise ValueError - if isinstance(yield_item, ExprName): - annotation = yield_item - elif yield_item.is_tuple and returns_multiple_items: - annotation = yield_item.slice.elements[index] - else: - annotation = yield_item + annotation = _maybe_destructure_annotation( + _unpack_generators(annotation, 0, mandatory=True), + index, + multiple=returns_multiple_items, + ) if annotation is None: yielded_value = repr(name) if name else index + 1 @@ -593,32 +627,23 @@ def _read_receives_section( ) -> tuple[DocstringSectionReceives | None, int]: receives = [] - if receives_multiple_items: - block, new_offset = _read_block_items(docstring, offset=offset, **options) - else: - one_block, new_offset = _read_block(docstring, offset=offset, **options) - block = [(new_offset, one_block.splitlines())] + block, new_offset = _read_block_items_maybe( + docstring, + offset=offset, + multiple=receives_multiple_items, + **options, + ) for index, (line_number, receive_lines) in enumerate(block): - if receives_multiple_items: - match = _RE_NAME_ANNOTATION_DESCRIPTION.match(receive_lines[0]) - if not match: - docstring_warning( - docstring, - line_number, - f"Failed to get name, annotation or description from '{receive_lines[0]}'", - ) - continue - name, annotation, description = match.groups() - else: - name = None - if ":" in receive_lines[0]: - annotation, description = receive_lines[0].split(":", 1) - annotation = annotation.lstrip("(").rstrip(")") - else: - annotation = None - description = receive_lines[0] - description = "\n".join([description.lstrip(), *receive_lines[1:]]).rstrip("\n") + try: + name, annotation, description = _get_name_annotation_description( + docstring, + line_number, + receive_lines, + named=receives_named_value, + ) + except ValueError: + continue if annotation: # try to compile the annotation to transform it into an expression @@ -627,14 +652,11 @@ def _read_receives_section( # try to retrieve the annotation from the docstring parent with suppress(AttributeError, KeyError): annotation = docstring.parent.returns # type: ignore[union-attr] - if annotation.is_generator: - receives_item = annotation.slice.elements[1] - if isinstance(receives_item, ExprName): - annotation = receives_item - elif receives_item.is_tuple and receives_multiple_items: - annotation = receives_item.slice.elements[index] - else: - annotation = receives_item + annotation = _maybe_destructure_annotation( + _unpack_generators(annotation, 1, mandatory=True), + index, + multiple=receives_multiple_items, + ) if annotation is None: received_value = repr(name) if name else index + 1 From f0b1278eb99bf0d08731dd3384aba1a8de6682bf Mon Sep 17 00:00:00 2001 From: Marco Ricci Date: Sun, 8 Sep 2024 13:00:46 +0200 Subject: [PATCH 3/3] docs: document non-multiple/non-named values in Yields and Receives --- docs/reference/docstrings.md | 72 +++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/docs/reference/docstrings.md b/docs/reference/docstrings.md index cac12a4d..1db818ae 100644 --- a/docs/reference/docstrings.md +++ b/docs/reference/docstrings.md @@ -141,9 +141,14 @@ The parser accepts a few options: - `ignore_init_summary`: Ignore the first line in `__init__` methods' docstrings. Useful when merging `__init__` docstring into class' docstrings with mkdocstrings-python's [`merge_init_into_class`][merge_init] option. Default: false. -- `returns_multiple_items`: Parse [Returns sections](#google-section-returns) as if they contain multiple items. +- `returns_multiple_items`: Parse [Returns sections](#google-section-returns) and [Yields sections](#google-section-yields) as if they contain multiple items. It means that continuation lines must be indented. Default: true. -- `returns_named_value`: Whether to parse `thing: Description` in [Returns sections](#google-section-returns) as a name and description, +- `returns_named_value`: Whether to parse `thing: Description` in [Returns sections](#google-section-returns) and [Yields sections](#google-section-yields) as a name and description, + rather than a type and description. When true, type must be wrapped in parentheses: `(int): Description.`. + When false, parentheses are optional but the items cannot be named: `int: Description`. Default: true. +- `receives_multiple_items`: Parse [Receives sections](#google-section-receives) as if they contain multiple items. + It means that continuation lines must be indented. Default: true. +- `receives_named_value`: Whether to parse `thing: Description` in [Receives sections](#google-section-receives) as a name and description, rather than a type and description. When true, type must be wrapped in parentheses: `(int): Description.`. When false, parentheses are optional but the items cannot be named: `int: Description`. Default: true. - `returns_type_in_property_summary`: Whether to parse the return type of properties @@ -580,6 +585,22 @@ def foo() -> Iterator[tuple[float, float, datetime]]: ... ``` +You have to indent each continuation line when documenting yielded values, +even if there's only one value yielded: + +```python +"""Foo. + +Yields: + partial_result: Some partial result. + A longer description of details and other information + for this partial result. +""" +``` + +If you don't want to indent continuation lines for the only yielded value, +use the [`returns_multiple_items=False`](#google-options) parser option. + Type annotations can as usual be overridden using types in parentheses in the docstring itself: @@ -593,6 +614,22 @@ Yields: """ ``` +If you want to specify the type without a name, you still have to wrap the type in parentheses: + +```python +"""Foo. + +Yields: + (int): Absissa. + (int): Ordinate. + (int): Timestamp. +""" +``` + +If you don't want to wrap the type in parentheses, +use the [`returns_named_value=False`](#google-options) parser option. +Setting it to false will disallow specifying a name. + TIP: **Types in docstrings are resolved using the docstrings' parent scope.** See previous tips for types in docstrings. @@ -663,6 +700,22 @@ def foo() -> Generator[int, tuple[str, bool], None]: ... ``` +You have to indent each continuation line when documenting received values, +even if there's only one value received: + +```python +"""Foo. + +Receives: + data: Input data. + A longer description of what this data actually is, + and what it isn't. +""" +``` + +If you don't want to indent continuation lines for the only received value, +use the [`receives_multiple_items=False`](#google-options) parser option. + Type annotations can as usual be overridden using types in parentheses in the docstring itself: @@ -675,6 +728,21 @@ Receives: """ ``` +If you want to specify the type without a name, you still have to wrap the type in parentheses: + +```python +"""Foo. + +Receives: + (ModeEnum): Some mode. + (int): Some flag. +""" +``` + +If you don't want to wrap the type in parentheses, +use the [`receives_named_value=False`](#google-options) parser option. +Setting it to false will disallow specifying a name. + TIP: **Types in docstrings are resolved using the docstrings' parent scope.** See previous tips for types in docstrings.