From 6afef4772b36b955c18237b34a22423a618483ad Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Thu, 3 Aug 2023 13:47:45 +0100 Subject: [PATCH] DSLParser: pass `function` as a parameter --- Tools/clinic/clinic.py | 258 ++++++++++++++++++++++++++--------------- 1 file changed, 162 insertions(+), 96 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1bcdb6b1c3640a..879572cf3b56a6 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4388,7 +4388,10 @@ def dedent(self, line: str) -> str: return line[indent:] -StateKeeper = Callable[[str], None] +class StateKeeper(Protocol): + def __call__(self, function: Function | None, line: str) -> Function | None: ... + + ConverterArgs = dict[str, Any] class ParamState(enum.IntEnum): @@ -4423,7 +4426,6 @@ class ParamState(enum.IntEnum): class DSLParser: - function: Function | None state: StateKeeper keyword_only: bool positional_only: bool @@ -4454,7 +4456,6 @@ def __init__(self, clinic: Clinic) -> None: self.reset() def reset(self) -> None: - self.function = None self.state = self.state_dsl_start self.keyword_only = False self.positional_only = False @@ -4605,16 +4606,17 @@ def parse(self, block: Block) -> None: block.output = [] block_start = self.clinic.block_parser.line_number lines = block.input.split('\n') + function: Function | None = None for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): if '\t' in line: fail('Tab characters are illegal in the Clinic DSL.\n\t' + repr(line), line_number=block_start) try: - self.state(line) + function = self.state(function, line) except ClinicError as exc: exc.lineno = line_number raise - self.do_post_block_processing_cleanup() + self.do_post_block_processing_cleanup(function) block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) if self.preserve_output: @@ -4648,15 +4650,20 @@ def calculate_indent(line: str) -> int: def next( self, state: StateKeeper, - line: str | None = None - ) -> None: + *, + function: Function | None, + line: str | None = None, + ) -> Function | None: self.state = state if line is not None: - self.state(line) + function = self.state(function=function, line=line) + return function + + def state_dsl_start(self, function: Function | None, line: str) -> Function | None: + assert function is None - def state_dsl_start(self, line: str) -> None: if not self.valid_line(line): - return + return None # is it a directive? fields = shlex.split(line) @@ -4667,11 +4674,13 @@ def state_dsl_start(self, line: str) -> None: directive(*fields[1:]) except TypeError as e: fail(str(e)) - return + return None - self.next(self.state_modulename_name, line) + return self.next(self.state_modulename_name, function=None, line=line) - def state_modulename_name(self, line: str) -> None: + def state_modulename_name( + self, function: Function | None, line: str + ) -> Function | None: # looking for declaration, which establishes the leftmost column # line should be # modulename.fnname [as c_basename] [-> return annotation] @@ -4688,8 +4697,10 @@ def state_modulename_name(self, line: str) -> None: # this line is permitted to start with whitespace. # we'll call this number of spaces F (for "function"). + assert function is None + if not self.valid_line(line): - return + return None self.indent.infer(line) @@ -4727,11 +4738,9 @@ def state_modulename_name(self, line: str) -> None: name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, docstring='' ) - self.function = function self.block.signatures.append(function) (cls or module).functions.append(function) - self.next(self.state_function_docstring) - return + return self.next(self.state_function_docstring, function=function) line, _, returns = line.partition('->') returns = returns.strip() @@ -4787,23 +4796,32 @@ def state_modulename_name(self, line: str) -> None: return_converter = CReturnConverter() if not module: - fail("Undefined module used in declaration of " + repr(full_name.strip()) + ".") - self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, - return_converter=return_converter, kind=self.kind, coexist=self.coexist) - self.block.signatures.append(self.function) + fail(f"Undefined module used in declaration of {full_name.strip()!r}.") + + function = Function( + name=function_name, + full_name=full_name, + module=module, + cls=cls, + c_basename=c_basename, + return_converter=return_converter, + kind=self.kind, + coexist=self.coexist + ) + self.block.signatures.append(function) # insert a self converter automatically - type, name = correct_name_for_self(self.function) + type, name = correct_name_for_self(function) kwargs = {} if cls and type == "PyObject *": kwargs['type'] = cls.typedef - sc = self.function.self_converter = self_converter(name, name, self.function, **kwargs) + sc = function.self_converter = self_converter(name, name, function, **kwargs) p_self = Parameter(name, inspect.Parameter.POSITIONAL_ONLY, - function=self.function, converter=sc) - self.function.parameters[name] = p_self + function=function, converter=sc) + function.parameters[name] = p_self - (cls or module).functions.append(self.function) - self.next(self.state_parameters_start) + (cls or module).functions.append(function) + return self.next(self.state_parameters_start, function=function) # Now entering the parameters section. The rules, formally stated: # @@ -4860,33 +4878,37 @@ def state_modulename_name(self, line: str) -> None: # separate boolean state variables.) The states are defined in the # ParamState class. - def state_parameters_start(self, line: str) -> None: - if not self.valid_line(line): - return - - # if this line is not indented, we have no parameters - if not self.indent.infer(line): - return self.next(self.state_function_docstring, line) + def state_parameters_start(self, function: Function | None, line: str) -> Function: + assert function is not None - self.parameter_continuation = '' - return self.next(self.state_parameter, line) + if self.valid_line(line): + # if this line is not indented, we have no parameters + if not self.indent.infer(line): + self.next( + self.state_function_docstring, function=function, line=line + ) + else: + self.parameter_continuation = '' + self.next(self.state_parameter, function=function, line=line) + return function - def to_required(self) -> None: + def to_required(self, function: Function) -> None: """ Transition to the "required" parameter state. """ if self.parameter_state is not ParamState.REQUIRED: self.parameter_state = ParamState.REQUIRED - assert self.function is not None - for p in self.function.parameters.values(): + for p in function.parameters.values(): p.group = -p.group - def state_parameter(self, line: str) -> None: - assert isinstance(self.function, Function) + def state_parameter( + self, function: Function | None, line: str + ) -> Function: + assert function is not None if not self.valid_line(line): - return + return function if self.parameter_continuation: line = self.parameter_continuation + ' ' + line.lstrip() @@ -4896,45 +4918,53 @@ def state_parameter(self, line: str) -> None: indent = self.indent.infer(line) if indent == -1: # we outdented, must be to definition column - return self.next(self.state_function_docstring, line) + self.next( + self.state_function_docstring, function=function, line=line + ) + return function if indent == 1: # we indented, must be to new parameter docstring column - return self.next(self.state_parameter_docstring_start, line) + self.next( + self.state_parameter_docstring_start, function=function, line=line + ) + return function line = line.rstrip() if line.endswith('\\'): self.parameter_continuation = line[:-1] - return + return function - func = self.function match line.lstrip(): case '*': - self.parse_star(func) + self.parse_star(function) case '[': - self.parse_opening_square_bracket(func) + self.parse_opening_square_bracket(function) case ']': - self.parse_closing_square_bracket(func) + self.parse_closing_square_bracket(function) case '/': - self.parse_slash(func) + self.parse_slash(function) case param: - self.parse_parameter(param) + self.parse_parameter(function, param) - def parse_parameter(self, line: str) -> None: - assert self.function is not None + return function + def parse_parameter(self, function: Function, line: str) -> None: match self.parameter_state: case ParamState.START | ParamState.REQUIRED: - self.to_required() + self.to_required(function) case ParamState.LEFT_SQUARE_BEFORE: self.parameter_state = ParamState.GROUP_BEFORE case ParamState.GROUP_BEFORE: if not self.group: - self.to_required() + self.to_required(function) case ParamState.GROUP_AFTER | ParamState.OPTIONAL: pass case st: - fail(f"Function {self.function.name} has an unsupported group configuration. (Unexpected state {st}.a)") + fail( + f"Function {function.name} has an unsupported group configuration. " + f"(Unexpected state {st}.a)" + ) # handle "as" for parameters too c_name = None @@ -4973,18 +5003,33 @@ def parse_parameter(self, line: str) -> None: except SyntaxError: pass if not module: - fail("Function " + self.function.name + " has an invalid parameter declaration:\n\t" + line) + fail( + f"Function {function.name} has an invalid parameter declaration:\n\t" + f"{line}" + ) - function = module.body[0] - assert isinstance(function, ast.FunctionDef) - function_args = function.args + function_node = module.body[0] + assert isinstance(function_node, ast.FunctionDef) + function_args = function_node.args if len(function_args.args) > 1: - fail("Function " + self.function.name + " has an invalid parameter declaration (comma?):\n\t" + line) + fail( + f"Function {function.name} " + f"has an invalid parameter declaration (comma?):\n\t" + f"{line}" + ) if function_args.defaults or function_args.kw_defaults: - fail("Function " + self.function.name + " has an invalid parameter declaration (default value?):\n\t" + line) + fail( + f"Function {function.name} " + f"has an invalid parameter declaration (default value?):\n\t" + f"{line}" + ) if function_args.kwarg: - fail("Function " + self.function.name + " has an invalid parameter declaration (**kwargs?):\n\t" + line) + fail( + f"Function {function.name} " + f"has an invalid parameter declaration (**kwargs?):\n\t" + f"{line}" + ) if function_args.vararg: is_vararg = True @@ -5129,7 +5174,9 @@ def bad_node(self, node: ast.AST) -> None: fail(f'{name} is not a valid {legacy_str}converter') # if you use a c_name for the parameter, we just give that name to the converter # but the parameter object gets the python name - converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) + converter = dict[name]( + c_name or parameter_name, parameter_name, function, value, **kwargs + ) kind: inspect._ParameterKind if is_vararg: @@ -5140,7 +5187,7 @@ def bad_node(self, node: ast.AST) -> None: kind = inspect.Parameter.POSITIONAL_OR_KEYWORD if isinstance(converter, self_converter): - if len(self.function.parameters) == 1: + if len(function.parameters) == 1: if self.parameter_state is not ParamState.REQUIRED: fail("A 'self' parameter cannot be marked optional.") if value is not unspecified: @@ -5149,12 +5196,12 @@ def bad_node(self, node: ast.AST) -> None: fail("A 'self' parameter cannot be in an optional group.") kind = inspect.Parameter.POSITIONAL_ONLY self.parameter_state = ParamState.START - self.function.parameters.clear() + function.parameters.clear() else: fail("A 'self' parameter, if specified, must be the very first thing in the parameter block.") if isinstance(converter, defining_class_converter): - _lp = len(self.function.parameters) + _lp = len(function.parameters) if _lp == 1: if self.parameter_state is not ParamState.REQUIRED: fail("A 'defining_class' parameter cannot be marked optional.") @@ -5165,17 +5212,23 @@ def bad_node(self, node: ast.AST) -> None: else: fail("A 'defining_class' parameter, if specified, must either be the first thing in the parameter block, or come just after 'self'.") + p = Parameter( + parameter_name, + kind, + function=function, + converter=converter, + default=value, + group=self.group + ) - p = Parameter(parameter_name, kind, function=self.function, converter=converter, default=value, group=self.group) - - names = [k.name for k in self.function.parameters.values()] + names = [k.name for k in function.parameters.values()] if parameter_name in names[1:]: fail("You can't have two parameters named " + repr(parameter_name) + "!") elif names and parameter_name == names[0] and c_name is None: fail(f"Parameter '{parameter_name}' requires a custom C name") key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name - self.function.parameters[key] = p + function.parameters[key] = p @staticmethod def parse_converter( @@ -5265,11 +5318,15 @@ def parse_slash(self, function: Function) -> None: "positional-only parameters, which is unsupported.") p.kind = inspect.Parameter.POSITIONAL_ONLY - def state_parameter_docstring_start(self, line: str) -> None: + def state_parameter_docstring_start( + self, function: Function | None, line: str + ) -> Function: + assert function is not None assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" self.parameter_docstring_indent = len(self.indent.margin) assert self.indent.depth == 3 - return self.next(self.state_parameter_docstring, line) + self.next(self.state_parameter_docstring, function=function, line=line) + return function def docstring_append(self, obj: Function | Parameter, line: str) -> None: """Add a rstripped line to the current docstring.""" @@ -5288,9 +5345,13 @@ def docstring_append(self, obj: Function | Parameter, line: str) -> None: # every line of the docstring must start with at least F spaces, # where F > P. # these F spaces will be stripped. - def state_parameter_docstring(self, line: str) -> None: + def state_parameter_docstring( + self, function: Function | None, line: str + ) -> Function: + assert function is not None + if not self.valid_line(line): - return + return function indent = self.indent.measure(line) if indent < self.parameter_docstring_indent: @@ -5298,30 +5359,34 @@ def state_parameter_docstring(self, line: str) -> None: assert self.indent.depth < 3 if self.indent.depth == 2: # back to a parameter - return self.next(self.state_parameter, line) + self.next(self.state_parameter, function=function, line=line) + return function assert self.indent.depth == 1 - return self.next(self.state_function_docstring, line) + self.next( + self.state_function_docstring, function=function, line=line + ) + return function - assert self.function and self.function.parameters - last_param = next(reversed(self.function.parameters.values())) + assert function.parameters + last_param = next(reversed(function.parameters.values())) self.docstring_append(last_param, line) + return function # the final stanza of the DSL is the docstring. - def state_function_docstring(self, line: str) -> None: - assert self.function is not None + def state_function_docstring( + self, function: Function | None, line: str + ) -> Function: + assert function is not None if self.group: - fail("Function " + self.function.name + " has a ] without a matching [.") - - if not self.valid_line(line): - return + fail(f"Function {function.name} has a ] without a matching [.") - self.docstring_append(self.function, line) + if self.valid_line(line): + self.docstring_append(function, line) - def format_docstring(self) -> str: - f = self.function - assert f is not None + return function + def format_docstring(self, f: Function) -> str: new_or_init = f.kind.new_or_init if new_or_init and not f.docstring: # don't render a docstring at all, no signature, nothing. @@ -5568,26 +5633,27 @@ def add_parameter(text: str) -> None: return docstring - def do_post_block_processing_cleanup(self) -> None: + def do_post_block_processing_cleanup(self, function: Function | None) -> None: """ Called when processing the block is done. """ - if not self.function: + if not function: return if self.keyword_only: - values = self.function.parameters.values() + values = function.parameters.values() if not values: no_parameter_after_star = True else: last_parameter = next(reversed(list(values))) no_parameter_after_star = last_parameter.kind != inspect.Parameter.KEYWORD_ONLY if no_parameter_after_star: - fail("Function " + self.function.name + " specifies '*' without any parameters afterwards.") - - self.function.docstring = self.format_docstring() - + fail( + f"Function {function.name} specifies '*' " + f"without any parameters afterwards." + ) + function.docstring = self.format_docstring(function) # maps strings to callables.