Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gh-113317: Add Codegen class to Argument Clinic #117626

Merged
merged 3 commits into from
Apr 11, 2024
Merged

gh-113317: Add Codegen class to Argument Clinic #117626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
869a484
gh-113317: Add Codegen class to Argument Clinic
vstinner Apr 8, 2024
a7479ba
Update test_clinic to fix the Lint CI
vstinner Apr 8, 2024
e446461
Add get_includes() methods
vstinner Apr 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions Lib/test/test_clinic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -822,9 +822,8 @@ def _test(self, input, output):

blocks = list(BlockParser(input, language))
writer = BlockPrinter(language)
c = _make_clinic()
for block in blocks:
writer.print_block(block, limited_capi=c.limited_capi, header_includes=c.includes)
writer.print_block(block)
output = writer.f.getvalue()
assert output == input, "output != input!\n\noutput " + repr(output) + "\n\n input " + repr(input)

Expand Down
40 changes: 7 additions & 33 deletions Tools/clinic/libclinic/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@
from libclinic import fail, warn
from libclinic.function import Class
from libclinic.block_parser import Block, BlockParser
from libclinic.crenderdata import Include
from libclinic.codegen import BlockPrinter, Destination
from libclinic.codegen import BlockPrinter, Destination, Codegen
from libclinic.parser import Parser, PythonParser
from libclinic.dsl_parser import DSLParser
if TYPE_CHECKING:
Expand Down Expand Up @@ -102,8 +101,7 @@ def __init__(
self.modules: ModuleDict = {}
self.classes: ClassDict = {}
self.functions: list[Function] = []
# dict: include name => Include instance
self.includes: dict[str, Include] = {}
self.codegen = Codegen(self.limited_capi)

self.line_prefix = self.line_suffix = ''

Expand Down Expand Up @@ -132,7 +130,6 @@ def __init__(
DestBufferList = list[DestBufferType]

self.destination_buffers_stack: DestBufferList = []
self.ifndef_symbols: set[str] = set()

self.presets: dict[str, dict[Any, Any]] = {}
preset = None
Expand All @@ -159,24 +156,6 @@ def __init__(
assert name in self.destination_buffers
preset[name] = buffer

def add_include(self, name: str, reason: str,
*, condition: str | None = None) -> None:
try:
existing = self.includes[name]
except KeyError:
pass
else:
if existing.condition and not condition:
# If the previous include has a condition and the new one is
# unconditional, override the include.
pass
else:
# Already included, do nothing. Only mention a single reason,
# no need to list all of them.
return

self.includes[name] = Include(name, reason, condition)

def add_destination(
self,
name: str,
Expand Down Expand Up @@ -212,9 +191,7 @@ def parse(self, input: str) -> str:
self.parsers[dsl_name] = parsers[dsl_name](self)
parser = self.parsers[dsl_name]
parser.parse(block)
printer.print_block(block,
limited_capi=self.limited_capi,
header_includes=self.includes)
printer.print_block(block)

# these are destinations not buffers
for name, destination in self.destinations.items():
Expand All @@ -229,9 +206,7 @@ def parse(self, input: str) -> str:
block.input = "dump " + name + "\n"
warn("Destination buffer " + repr(name) + " not empty at end of file, emptying.")
printer.write("\n")
printer.print_block(block,
limited_capi=self.limited_capi,
header_includes=self.includes)
printer.print_block(block)
continue

if destination.type == 'file':
Expand All @@ -255,11 +230,10 @@ def parse(self, input: str) -> str:
pass

block.input = 'preserve\n'
includes = self.codegen.get_includes()

printer_2 = BlockPrinter(self.language)
printer_2.print_block(block,
core_includes=True,
limited_capi=self.limited_capi,
header_includes=self.includes)
printer_2.print_block(block, header_includes=includes)
libclinic.write_file(destination.filename,
printer_2.f.getvalue())
continue
Expand Down
100 changes: 48 additions & 52 deletions Tools/clinic/libclinic/clanguage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
unspecified, fail, warn, Sentinels, VersionTuple)
from libclinic.function import (
GETTER, SETTER, METHOD_INIT, METHOD_NEW)
from libclinic.crenderdata import CRenderData, TemplateDict
from libclinic.codegen import CRenderData, TemplateDict, Codegen
from libclinic.language import Language
from libclinic.function import (
Module, Class, Function, Parameter,
Expand All @@ -26,15 +26,15 @@ def declare_parser(
f: Function,
*,
hasformat: bool = False,
clinic: Clinic,
limited_capi: bool,
codegen: Codegen,
) -> str:
"""
Generates the code template for a static local PyArg_Parser variable,
with an initializer. For core code (incl. builtin modules) the
kwtuple field is also statically initialized. Otherwise
it is initialized at runtime.
"""
limited_capi = codegen.limited_capi
if hasformat:
fname = ''
format_ = '.format = "{format_units}:{name}",'
Expand Down Expand Up @@ -80,8 +80,8 @@ def declare_parser(
""" % num_keywords

condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)'
clinic.add_include('pycore_gc.h', 'PyGC_Head', condition=condition)
clinic.add_include('pycore_runtime.h', '_Py_ID()', condition=condition)
codegen.add_include('pycore_gc.h', 'PyGC_Head', condition=condition)
codegen.add_include('pycore_runtime.h', '_Py_ID()', condition=condition)

declarations += """
static const char * const _keywords[] = {{{keywords_c} NULL}};
Expand Down Expand Up @@ -317,14 +317,14 @@ def deprecate_keyword_use(
self,
func: Function,
params: dict[int, Parameter],
argname_fmt: str | None,
argname_fmt: str | None = None,
*,
fastcall: bool,
limited_capi: bool,
clinic: Clinic,
codegen: Codegen,
) -> str:
assert len(params) > 0
last_param = next(reversed(params.values()))
limited_capi = codegen.limited_capi

# Format the deprecation message.
containscheck = ""
Expand All @@ -336,11 +336,11 @@ def deprecate_keyword_use(
elif fastcall:
conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, &_Py_ID({p.name}))")
containscheck = "PySequence_Contains"
clinic.add_include('pycore_runtime.h', '_Py_ID()')
codegen.add_include('pycore_runtime.h', '_Py_ID()')
else:
conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))")
containscheck = "PyDict_Contains"
clinic.add_include('pycore_runtime.h', '_Py_ID()')
codegen.add_include('pycore_runtime.h', '_Py_ID()')
else:
conditions = [f"nargs < {i+1}"]
condition = ") || (".join(conditions)
Expand Down Expand Up @@ -399,7 +399,7 @@ def deprecate_keyword_use(
def output_templates(
self,
f: Function,
clinic: Clinic
codegen: Codegen,
) -> dict[str, str]:
parameters = list(f.parameters.values())
assert parameters
Expand All @@ -412,7 +412,7 @@ def output_templates(
converters = [p.converter for p in parameters]

if f.critical_section:
clinic.add_include('pycore_critical_section.h', 'Py_BEGIN_CRITICAL_SECTION()')
codegen.add_include('pycore_critical_section.h', 'Py_BEGIN_CRITICAL_SECTION()')
has_option_groups = parameters and (parameters[0].group or parameters[-1].group)
simple_return = (f.return_converter.type == 'PyObject *'
and not f.critical_section)
Expand Down Expand Up @@ -517,7 +517,7 @@ def parser_body(
parser_declarations=declarations)

fastcall = not new_or_init
limited_capi = clinic.limited_capi
limited_capi = codegen.limited_capi
if limited_capi and (pseudo_args or
(any(p.is_optional() for p in parameters) and
any(p.is_keyword_only() and not p.is_optional() for p in parameters)) or
Expand Down Expand Up @@ -673,8 +673,8 @@ def parser_body(
""",
indent=4))
else:
clinic.add_include('pycore_modsupport.h',
'_PyArg_CheckPositional()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_CheckPositional()')
parser_code = [libclinic.normalize_snippet(f"""
if (!_PyArg_CheckPositional("{{name}}", {nargs}, {min_pos}, {max_args})) {{{{
goto exit;
Expand Down Expand Up @@ -735,8 +735,8 @@ def parser_body(
if limited_capi:
fastcall = False
if fastcall:
clinic.add_include('pycore_modsupport.h',
'_PyArg_ParseStack()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_ParseStack()')
parser_code = [libclinic.normalize_snippet("""
if (!_PyArg_ParseStack(args, nargs, "{format_units}:{name}",
{parse_arguments})) {{
Expand Down Expand Up @@ -773,17 +773,17 @@ def parser_body(
fastcall = False
else:
if vararg == self.NO_VARARG:
clinic.add_include('pycore_modsupport.h',
'_PyArg_UnpackKeywords()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_UnpackKeywords()')
args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % (
min_pos,
max_pos,
min_kw_only
)
nargs = "nargs"
else:
clinic.add_include('pycore_modsupport.h',
'_PyArg_UnpackKeywordsWithVararg()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_UnpackKeywordsWithVararg()')
args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % (
min_pos,
max_pos,
Expand All @@ -796,8 +796,7 @@ def parser_body(
flags = "METH_FASTCALL|METH_KEYWORDS"
parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS
argname_fmt = 'args[%d]'
declarations = declare_parser(f, clinic=clinic,
limited_capi=clinic.limited_capi)
declarations = declare_parser(f, codegen=codegen)
declarations += "\nPyObject *argsbuf[%s];" % len(converters)
if has_optional_kw:
declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only)
Expand All @@ -812,8 +811,7 @@ def parser_body(
flags = "METH_VARARGS|METH_KEYWORDS"
parser_prototype = self.PARSER_PROTOTYPE_KEYWORD
argname_fmt = 'fastargs[%d]'
declarations = declare_parser(f, clinic=clinic,
limited_capi=clinic.limited_capi)
declarations = declare_parser(f, codegen=codegen)
declarations += "\nPyObject *argsbuf[%s];" % len(converters)
declarations += "\nPyObject * const *fastargs;"
declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);"
Expand All @@ -832,10 +830,10 @@ def parser_body(

if parser_code is not None:
if deprecated_keywords:
code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt,
clinic=clinic,
fastcall=fastcall,
limited_capi=limited_capi)
code = self.deprecate_keyword_use(f, deprecated_keywords,
argname_fmt,
codegen=codegen,
fastcall=fastcall)
parser_code.append(code)

add_label: str | None = None
Expand Down Expand Up @@ -903,9 +901,8 @@ def parser_body(
for parameter in parameters:
parameter.converter.use_converter()

declarations = declare_parser(f, clinic=clinic,
hasformat=True,
limited_capi=limited_capi)
declarations = declare_parser(f, codegen=codegen,
hasformat=True)
if limited_capi:
# positional-or-keyword arguments
assert not fastcall
Expand All @@ -921,17 +918,17 @@ def parser_body(
declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);"

elif fastcall:
clinic.add_include('pycore_modsupport.h',
'_PyArg_ParseStackAndKeywords()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_ParseStackAndKeywords()')
parser_code = [libclinic.normalize_snippet("""
if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma}
{parse_arguments})) {{
goto exit;
}}
""", indent=4)]
else:
clinic.add_include('pycore_modsupport.h',
'_PyArg_ParseTupleAndKeywordsFast()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_ParseTupleAndKeywordsFast()')
parser_code = [libclinic.normalize_snippet("""
if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser,
{parse_arguments})) {{
Expand All @@ -941,10 +938,9 @@ def parser_body(
if deprecated_positionals or deprecated_keywords:
declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);"
if deprecated_keywords:
code = self.deprecate_keyword_use(f, deprecated_keywords, None,
clinic=clinic,
fastcall=fastcall,
limited_capi=limited_capi)
code = self.deprecate_keyword_use(f, deprecated_keywords,
codegen=codegen,
fastcall=fastcall)
parser_code.append(code)

if deprecated_positionals:
Expand All @@ -960,9 +956,9 @@ def parser_body(
# Copy includes from parameters to Clinic after parse_arg() has been
# called above.
for converter in converters:
for include in converter.includes:
clinic.add_include(include.filename, include.reason,
condition=include.condition)
for include in converter.get_includes():
codegen.add_include(include.filename, include.reason,
condition=include.condition)

if new_or_init:
methoddef_define = ''
Expand All @@ -984,16 +980,16 @@ def parser_body(

if not parses_keywords:
declarations = '{base_type_ptr}'
clinic.add_include('pycore_modsupport.h',
'_PyArg_NoKeywords()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_NoKeywords()')
fields.insert(0, libclinic.normalize_snippet("""
if ({self_type_check}!_PyArg_NoKeywords("{name}", kwargs)) {{
goto exit;
}}
""", indent=4))
if not parses_positional:
clinic.add_include('pycore_modsupport.h',
'_PyArg_NoPositional()')
codegen.add_include('pycore_modsupport.h',
'_PyArg_NoPositional()')
fields.insert(0, libclinic.normalize_snippet("""
if ({self_type_check}!_PyArg_NoPositional("{name}", args)) {{
goto exit;
Expand Down Expand Up @@ -1030,8 +1026,7 @@ def parser_body(
cpp_if = "#if " + conditional
cpp_endif = "#endif /* " + conditional + " */"

if methoddef_define and f.full_name not in clinic.ifndef_symbols:
clinic.ifndef_symbols.add(f.full_name)
if methoddef_define and codegen.add_ifndef_symbol(f.full_name):
methoddef_ifndef = self.METHODDEF_PROTOTYPE_IFNDEF

# add ';' to the end of parser_prototype and impl_prototype
Expand Down Expand Up @@ -1190,16 +1185,17 @@ def render_function(
clinic: Clinic,
f: Function | None
) -> str:
if f is None or clinic is None:
if f is None:
return ""

codegen = clinic.codegen
data = CRenderData()

assert f.parameters, "We should always have a 'self' at this point!"
parameters = f.render_parameters
converters = [p.converter for p in parameters]

templates = self.output_templates(f, clinic)
templates = self.output_templates(f, codegen)

f_self = parameters[0]
selfless = parameters[1:]
Expand Down Expand Up @@ -1323,7 +1319,7 @@ def render_function(

if has_option_groups:
self.render_option_group_parsing(f, template_dict,
limited_capi=clinic.limited_capi)
limited_capi=codegen.limited_capi)

# buffers, not destination
for name, destination in clinic.destination_buffers.items():
Expand Down
Loading
Loading