From 564fb56e2047b33922c8f5b64f4e26f7eb5eecb6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2024 15:37:22 +0100 Subject: [PATCH] gh-113317, AC: Add libclinic.function (#116807) Move Module, Class, Function and Parameter classes to a new libclinic.function module. Move VersionTuple and Sentinels to libclinic.utils. --- Tools/clinic/clinic.py | 248 +---------------------------- Tools/clinic/libclinic/__init__.py | 8 + Tools/clinic/libclinic/function.py | 237 +++++++++++++++++++++++++++ Tools/clinic/libclinic/utils.py | 18 ++- 4 files changed, 270 insertions(+), 241 deletions(-) create mode 100644 Tools/clinic/libclinic/function.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index fb56bd203749f98..6488d913168319d 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -12,7 +12,6 @@ import builtins as bltns import collections import contextlib -import copy import dataclasses as dc import enum import functools @@ -50,7 +49,14 @@ # Local imports. import libclinic import libclinic.cpp -from libclinic import ClinicError, fail, warn +from libclinic import ( + ClinicError, Sentinels, VersionTuple, + fail, warn, unspecified, unknown) +from libclinic.function import ( + Module, Class, Function, Parameter, + ClassDict, ModuleDict, FunctionKind, + CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, + GETTER, SETTER) # TODO: @@ -70,18 +76,6 @@ LIMITED_CAPI_REGEX = re.compile(r'# *define +Py_LIMITED_API') -class Sentinels(enum.Enum): - unspecified = "unspecified" - unknown = "unknown" - - def __repr__(self) -> str: - return f"<{self.value.capitalize()}>" - - -unspecified: Final = Sentinels.unspecified -unknown: Final = Sentinels.unknown - - # This one needs to be a distinct class, unlike the other two class Null: def __repr__(self) -> str: @@ -2096,9 +2090,7 @@ def dump(self) -> str: extensions['py'] = PythonLanguage -ClassDict = dict[str, "Class"] DestinationDict = dict[str, Destination] -ModuleDict = dict[str, "Module"] class Parser(Protocol): @@ -2418,38 +2410,6 @@ def parse(self, block: Block) -> None: block.output = s.getvalue() -@dc.dataclass(repr=False) -class Module: - name: str - module: Module | Clinic - - def __post_init__(self) -> None: - self.parent = self.module - self.modules: ModuleDict = {} - self.classes: ClassDict = {} - self.functions: list[Function] = [] - - def __repr__(self) -> str: - return "" - - -@dc.dataclass(repr=False) -class Class: - name: str - module: Module | Clinic - cls: Class | None - typedef: str - type_object: str - - def __post_init__(self) -> None: - self.parent = self.cls or self.module - self.classes: ClassDict = {} - self.functions: list[Function] = [] - - def __repr__(self) -> str: - return "" - - unsupported_special_methods: set[str] = set(""" __abs__ @@ -2522,201 +2482,9 @@ def __repr__(self) -> str: """.strip().split()) -class FunctionKind(enum.Enum): - INVALID = enum.auto() - CALLABLE = enum.auto() - STATIC_METHOD = enum.auto() - CLASS_METHOD = enum.auto() - METHOD_INIT = enum.auto() - METHOD_NEW = enum.auto() - GETTER = enum.auto() - SETTER = enum.auto() - - @functools.cached_property - def new_or_init(self) -> bool: - return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} - - def __repr__(self) -> str: - return f"" - - -INVALID: Final = FunctionKind.INVALID -CALLABLE: Final = FunctionKind.CALLABLE -STATIC_METHOD: Final = FunctionKind.STATIC_METHOD -CLASS_METHOD: Final = FunctionKind.CLASS_METHOD -METHOD_INIT: Final = FunctionKind.METHOD_INIT -METHOD_NEW: Final = FunctionKind.METHOD_NEW -GETTER: Final = FunctionKind.GETTER -SETTER: Final = FunctionKind.SETTER - -ParamDict = dict[str, "Parameter"] ReturnConverterType = Callable[..., "CReturnConverter"] -@dc.dataclass(repr=False) -class Function: - """ - Mutable duck type for inspect.Function. - - docstring - a str containing - * embedded line breaks - * text outdented to the left margin - * no trailing whitespace. - It will always be true that - (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) - """ - parameters: ParamDict = dc.field(default_factory=dict) - _: dc.KW_ONLY - name: str - module: Module | Clinic - cls: Class | None - c_basename: str - full_name: str - return_converter: CReturnConverter - kind: FunctionKind - coexist: bool - return_annotation: object = inspect.Signature.empty - docstring: str = '' - # docstring_only means "don't generate a machine-readable - # signature, just a normal docstring". it's True for - # functions with optional groups because we can't represent - # those accurately with inspect.Signature in 3.4. - docstring_only: bool = False - critical_section: bool = False - target_critical_section: list[str] = dc.field(default_factory=list) - - def __post_init__(self) -> None: - self.parent = self.cls or self.module - self.self_converter: self_converter | None = None - self.__render_parameters__: list[Parameter] | None = None - - @functools.cached_property - def displayname(self) -> str: - """Pretty-printable name.""" - if self.kind.new_or_init: - assert isinstance(self.cls, Class) - return self.cls.name - else: - return self.name - - @functools.cached_property - def fulldisplayname(self) -> str: - parent: Class | Module | Clinic | None - if self.kind.new_or_init: - parent = getattr(self.cls, "parent", None) - else: - parent = self.parent - name = self.displayname - while isinstance(parent, (Module, Class)): - name = f"{parent.name}.{name}" - parent = parent.parent - return name - - @property - def render_parameters(self) -> list[Parameter]: - if not self.__render_parameters__: - l: list[Parameter] = [] - self.__render_parameters__ = l - for p in self.parameters.values(): - p = p.copy() - p.converter.pre_render() - l.append(p) - return self.__render_parameters__ - - @property - def methoddef_flags(self) -> str | None: - if self.kind.new_or_init: - return None - flags = [] - match self.kind: - case FunctionKind.CLASS_METHOD: - flags.append('METH_CLASS') - case FunctionKind.STATIC_METHOD: - flags.append('METH_STATIC') - case _ as kind: - acceptable_kinds = {FunctionKind.CALLABLE, FunctionKind.GETTER, FunctionKind.SETTER} - assert kind in acceptable_kinds, f"unknown kind: {kind!r}" - if self.coexist: - flags.append('METH_COEXIST') - return '|'.join(flags) - - def __repr__(self) -> str: - return f'' - - def copy(self, **overrides: Any) -> Function: - f = dc.replace(self, **overrides) - f.parameters = { - name: value.copy(function=f) - for name, value in f.parameters.items() - } - return f - - -VersionTuple = tuple[int, int] - - -@dc.dataclass(repr=False, slots=True) -class Parameter: - """ - Mutable duck type of inspect.Parameter. - """ - name: str - kind: inspect._ParameterKind - _: dc.KW_ONLY - default: object = inspect.Parameter.empty - function: Function - converter: CConverter - annotation: object = inspect.Parameter.empty - docstring: str = '' - group: int = 0 - # (`None` signifies that there is no deprecation) - deprecated_positional: VersionTuple | None = None - deprecated_keyword: VersionTuple | None = None - right_bracket_count: int = dc.field(init=False, default=0) - - def __repr__(self) -> str: - return f'' - - def is_keyword_only(self) -> bool: - return self.kind == inspect.Parameter.KEYWORD_ONLY - - def is_positional_only(self) -> bool: - return self.kind == inspect.Parameter.POSITIONAL_ONLY - - def is_vararg(self) -> bool: - return self.kind == inspect.Parameter.VAR_POSITIONAL - - def is_optional(self) -> bool: - return not self.is_vararg() and (self.default is not unspecified) - - def copy( - self, - /, - *, - converter: CConverter | None = None, - function: Function | None = None, - **overrides: Any - ) -> Parameter: - function = function or self.function - if not converter: - converter = copy.copy(self.converter) - converter.function = function - return dc.replace(self, **overrides, function=function, converter=converter) - - def get_displayname(self, i: int) -> str: - if i == 0: - return 'argument' - if not self.is_positional_only(): - return f'argument {self.name!r}' - else: - return f'argument {i}' - - def render_docstring(self) -> str: - lines = [f" {self.name}"] - lines.extend(f" {line}" for line in self.docstring.split("\n")) - return "\n".join(lines).rstrip() - - CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) def add_c_converter( diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index 8efaad6539d7aee..32231b82bfdc076 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -28,6 +28,10 @@ compute_checksum, create_regex, write_file, + VersionTuple, + Sentinels, + unspecified, + unknown, ) @@ -60,6 +64,10 @@ "compute_checksum", "create_regex", "write_file", + "VersionTuple", + "Sentinels", + "unspecified", + "unknown", ] diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py new file mode 100644 index 000000000000000..48cb7d05a7caef3 --- /dev/null +++ b/Tools/clinic/libclinic/function.py @@ -0,0 +1,237 @@ +from __future__ import annotations +import dataclasses as dc +import copy +import enum +import functools +import inspect +from typing import Final, Any, TYPE_CHECKING +if TYPE_CHECKING: + from clinic import Clinic, CConverter, CReturnConverter, self_converter + +from libclinic import VersionTuple, unspecified + + +ClassDict = dict[str, "Class"] +ModuleDict = dict[str, "Module"] +ParamDict = dict[str, "Parameter"] + + +@dc.dataclass(repr=False) +class Module: + name: str + module: Module | Clinic + + def __post_init__(self) -> None: + self.parent = self.module + self.modules: ModuleDict = {} + self.classes: ClassDict = {} + self.functions: list[Function] = [] + + def __repr__(self) -> str: + return "" + + +@dc.dataclass(repr=False) +class Class: + name: str + module: Module | Clinic + cls: Class | None + typedef: str + type_object: str + + def __post_init__(self) -> None: + self.parent = self.cls or self.module + self.classes: ClassDict = {} + self.functions: list[Function] = [] + + def __repr__(self) -> str: + return "" + + +class FunctionKind(enum.Enum): + INVALID = enum.auto() + CALLABLE = enum.auto() + STATIC_METHOD = enum.auto() + CLASS_METHOD = enum.auto() + METHOD_INIT = enum.auto() + METHOD_NEW = enum.auto() + GETTER = enum.auto() + SETTER = enum.auto() + + @functools.cached_property + def new_or_init(self) -> bool: + return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} + + def __repr__(self) -> str: + return f"" + + +INVALID: Final = FunctionKind.INVALID +CALLABLE: Final = FunctionKind.CALLABLE +STATIC_METHOD: Final = FunctionKind.STATIC_METHOD +CLASS_METHOD: Final = FunctionKind.CLASS_METHOD +METHOD_INIT: Final = FunctionKind.METHOD_INIT +METHOD_NEW: Final = FunctionKind.METHOD_NEW +GETTER: Final = FunctionKind.GETTER +SETTER: Final = FunctionKind.SETTER + + +@dc.dataclass(repr=False) +class Function: + """ + Mutable duck type for inspect.Function. + + docstring - a str containing + * embedded line breaks + * text outdented to the left margin + * no trailing whitespace. + It will always be true that + (not docstring) or ((not docstring[0].isspace()) and (docstring.rstrip() == docstring)) + """ + parameters: ParamDict = dc.field(default_factory=dict) + _: dc.KW_ONLY + name: str + module: Module | Clinic + cls: Class | None + c_basename: str + full_name: str + return_converter: CReturnConverter + kind: FunctionKind + coexist: bool + return_annotation: object = inspect.Signature.empty + docstring: str = '' + # docstring_only means "don't generate a machine-readable + # signature, just a normal docstring". it's True for + # functions with optional groups because we can't represent + # those accurately with inspect.Signature in 3.4. + docstring_only: bool = False + critical_section: bool = False + target_critical_section: list[str] = dc.field(default_factory=list) + + def __post_init__(self) -> None: + self.parent = self.cls or self.module + self.self_converter: self_converter | None = None + self.__render_parameters__: list[Parameter] | None = None + + @functools.cached_property + def displayname(self) -> str: + """Pretty-printable name.""" + if self.kind.new_or_init: + assert isinstance(self.cls, Class) + return self.cls.name + else: + return self.name + + @functools.cached_property + def fulldisplayname(self) -> str: + parent: Class | Module | Clinic | None + if self.kind.new_or_init: + parent = getattr(self.cls, "parent", None) + else: + parent = self.parent + name = self.displayname + while isinstance(parent, (Module, Class)): + name = f"{parent.name}.{name}" + parent = parent.parent + return name + + @property + def render_parameters(self) -> list[Parameter]: + if not self.__render_parameters__: + l: list[Parameter] = [] + self.__render_parameters__ = l + for p in self.parameters.values(): + p = p.copy() + p.converter.pre_render() + l.append(p) + return self.__render_parameters__ + + @property + def methoddef_flags(self) -> str | None: + if self.kind.new_or_init: + return None + flags = [] + match self.kind: + case FunctionKind.CLASS_METHOD: + flags.append('METH_CLASS') + case FunctionKind.STATIC_METHOD: + flags.append('METH_STATIC') + case _ as kind: + acceptable_kinds = {FunctionKind.CALLABLE, FunctionKind.GETTER, FunctionKind.SETTER} + assert kind in acceptable_kinds, f"unknown kind: {kind!r}" + if self.coexist: + flags.append('METH_COEXIST') + return '|'.join(flags) + + def __repr__(self) -> str: + return f'' + + def copy(self, **overrides: Any) -> Function: + f = dc.replace(self, **overrides) + f.parameters = { + name: value.copy(function=f) + for name, value in f.parameters.items() + } + return f + + +@dc.dataclass(repr=False, slots=True) +class Parameter: + """ + Mutable duck type of inspect.Parameter. + """ + name: str + kind: inspect._ParameterKind + _: dc.KW_ONLY + default: object = inspect.Parameter.empty + function: Function + converter: CConverter + annotation: object = inspect.Parameter.empty + docstring: str = '' + group: int = 0 + # (`None` signifies that there is no deprecation) + deprecated_positional: VersionTuple | None = None + deprecated_keyword: VersionTuple | None = None + right_bracket_count: int = dc.field(init=False, default=0) + + def __repr__(self) -> str: + return f'' + + def is_keyword_only(self) -> bool: + return self.kind == inspect.Parameter.KEYWORD_ONLY + + def is_positional_only(self) -> bool: + return self.kind == inspect.Parameter.POSITIONAL_ONLY + + def is_vararg(self) -> bool: + return self.kind == inspect.Parameter.VAR_POSITIONAL + + def is_optional(self) -> bool: + return not self.is_vararg() and (self.default is not unspecified) + + def copy( + self, + /, + *, + converter: CConverter | None = None, + function: Function | None = None, + **overrides: Any + ) -> Parameter: + function = function or self.function + if not converter: + converter = copy.copy(self.converter) + converter.function = function + return dc.replace(self, **overrides, function=function, converter=converter) + + def get_displayname(self, i: int) -> str: + if i == 0: + return 'argument' + if not self.is_positional_only(): + return f'argument {self.name!r}' + else: + return f'argument {i}' + + def render_docstring(self) -> str: + lines = [f" {self.name}"] + lines.extend(f" {line}" for line in self.docstring.split("\n")) + return "\n".join(lines).rstrip() diff --git a/Tools/clinic/libclinic/utils.py b/Tools/clinic/libclinic/utils.py index d2d09387a73d1ea..95a69f70c5499d3 100644 --- a/Tools/clinic/libclinic/utils.py +++ b/Tools/clinic/libclinic/utils.py @@ -1,9 +1,10 @@ import collections +import enum import hashlib import os import re import string -from typing import Literal +from typing import Literal, Final def write_file(filename: str, new_contents: str) -> None: @@ -66,3 +67,18 @@ def get_value( ) -> Literal[""]: self.counts[key] += 1 return "" + + +VersionTuple = tuple[int, int] + + +class Sentinels(enum.Enum): + unspecified = "unspecified" + unknown = "unknown" + + def __repr__(self) -> str: + return f"<{self.value.capitalize()}>" + + +unspecified: Final = Sentinels.unspecified +unknown: Final = Sentinels.unknown