diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index c8a734ecad8e7b..8e91936680fab8 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -105,8 +105,9 @@ we also call *flavours*: PurePosixPath('setup.py') Each element of *pathsegments* can be either a string representing a - path segment, an object implementing the :class:`os.PathLike` interface - which returns a string, or another path object:: + path segment, or an object implementing the :class:`os.PathLike` interface + where the :meth:`~os.PathLike.__fspath__` method returns a string, + such as another path object:: >>> PurePath('foo', 'some/path', 'bar') PurePosixPath('foo/some/path/bar') diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 1240f80b0f11f0..550f808a16dfaa 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -530,12 +530,14 @@ class`. In addition, it provides a few more methods: is ``False``. The default values can be used to conveniently turn an integer into a - single byte object. However, when using the default arguments, don't try - to convert a value greater than 255 or you'll get an :exc:`OverflowError`:: + single byte object:: >>> (65).to_bytes() b'A' + However, when using the default arguments, don't try + to convert a value greater than 255 or you'll get an :exc:`OverflowError`. + Equivalent to:: def to_bytes(n, length=1, byteorder='big', signed=False): diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 3c759648e4e626..c60b4da75d1acb 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -540,7 +540,7 @@ The :mod:`test.support` module defines the following functions: Get size of a page in bytes. - .. versionadded:: 3.12 + .. versionadded:: 3.12 .. function:: setswitchinterval(interval) diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index e5dc82cf0dc2d9..5af56433f41573 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -24,6 +24,7 @@ from docutils.parsers.rst import directives from docutils.parsers.rst import Directive from docutils.statemachine import StringList +from sphinx.locale import _ as sphinx_gettext import csv from sphinx import addnodes @@ -168,11 +169,11 @@ def add_annotations(self, app, doctree): elif not entry.result_type.endswith("Object*"): continue if entry.result_refs is None: - rc = 'Return value: Always NULL.' + rc = sphinx_gettext('Return value: Always NULL.') elif entry.result_refs: - rc = 'Return value: New reference.' + rc = sphinx_gettext('Return value: New reference.') else: - rc = 'Return value: Borrowed reference.' + rc = sphinx_gettext('Return value: Borrowed reference.') node.insert(0, nodes.emphasis(rc, rc, classes=['refcount'])) diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 3438b44377fcb9..bab4aaeb4604b8 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -7,6 +7,11 @@ {% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} {% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %} +In extensions/c_annotations.py: + +{% trans %}Return value: Always NULL.{% endtrans %} +{% trans %}Return value: New reference.{% endtrans %} +{% trans %}Return value: Borrowed reference.{% endtrans %} In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 8fa8d250d533c9..8936bd381c9d97 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -326,6 +326,11 @@ also be used to improve performance. Enable C-level code profiling with ``gprof`` (disabled by default). +.. cmdoption:: --with-strict-overflow + + Add ``-fstrict-overflow`` to the C compiler flags (by default we add + ``-fno-strict-overflow`` instead). + .. _debug-build: diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index bffb8d03aa7cbb..10fcfb6a0b5639 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1317,14 +1317,17 @@ This section covers specific optimizations independent of the Faster CPython ============== -CPython 3.11 is on average `25% faster `_ -than CPython 3.10 when measured with the +CPython 3.11 is an average of +`25% faster `_ +than CPython 3.10 as measured with the `pyperformance `_ benchmark suite, -and compiled with GCC on Ubuntu Linux. Depending on your workload, the speedup -could be up to 10-60% faster. +when compiled with GCC on Ubuntu Linux. +Depending on your workload, the overall speedup could be 10-60%. -This project focuses on two major areas in Python: faster startup and faster -runtime. Other optimizations not under this project are listed in `Optimizations`_. +This project focuses on two major areas in Python: +:ref:`whatsnew311-faster-startup` and :ref:`whatsnew311-faster-runtime`. +Optimizations not covered by this project are listed separately under +:ref:`whatsnew311-optimizations`. .. _whatsnew311-faster-startup: @@ -1337,8 +1340,8 @@ Faster Startup Frozen imports / Static code objects ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python caches bytecode in the :ref:`__pycache__` directory to -speed up module loading. +Python caches :term:`bytecode` in the :ref:`__pycache__ ` +directory to speed up module loading. Previously in 3.10, Python module execution looked like this: @@ -1347,8 +1350,9 @@ Previously in 3.10, Python module execution looked like this: Read __pycache__ -> Unmarshal -> Heap allocated code object -> Evaluate In Python 3.11, the core modules essential for Python startup are "frozen". -This means that their code objects (and bytecode) are statically allocated -by the interpreter. This reduces the steps in module execution process to this: +This means that their :ref:`codeobjects` (and bytecode) +are statically allocated by the interpreter. +This reduces the steps in module execution process to: .. code-block:: text @@ -1357,7 +1361,7 @@ by the interpreter. This reduces the steps in module execution process to this: Interpreter startup is now 10-15% faster in Python 3.11. This has a big impact for short-running programs using Python. -(Contributed by Eric Snow, Guido van Rossum and Kumar Aditya in numerous issues.) +(Contributed by Eric Snow, Guido van Rossum and Kumar Aditya in many issues.) .. _whatsnew311-faster-runtime: @@ -1370,17 +1374,19 @@ Faster Runtime Cheaper, lazy Python frames ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python frames are created whenever Python calls a Python function. This frame -holds execution information. The following are new frame optimizations: +Python frames, holding execution information, +are created whenever Python calls a Python function. +The following are new frame optimizations: - Streamlined the frame creation process. - Avoided memory allocation by generously re-using frame space on the C stack. - Streamlined the internal frame struct to contain only essential information. Frames previously held extra debugging and memory management information. -Old-style frame objects are now created only when requested by debuggers or -by Python introspection functions such as ``sys._getframe`` or -``inspect.currentframe``. For most user code, no frame objects are +Old-style :ref:`frame objects ` +are now created only when requested by debuggers +or by Python introspection functions such as :func:`sys._getframe` and +:func:`inspect.currentframe`. For most user code, no frame objects are created at all. As a result, nearly all Python functions calls have sped up significantly. We measured a 3-7% speedup in pyperformance. @@ -1401,10 +1407,11 @@ In 3.11, when CPython detects Python code calling another Python function, it sets up a new frame, and "jumps" to the new code inside the new frame. This avoids calling the C interpreting function altogether. -Most Python function calls now consume no C stack space. This speeds up -most of such calls. In simple recursive functions like fibonacci or -factorial, a 1.7x speedup was observed. This also means recursive functions -can recurse significantly deeper (if the user increases the recursion limit). +Most Python function calls now consume no C stack space, speeding them up. +In simple recursive functions like fibonacci or +factorial, we observed a 1.7x speedup. This also means recursive functions +can recurse significantly deeper +(if the user increases the recursion limit with :func:`sys.setrecursionlimit`). We measured a 1-3% improvement in pyperformance. (Contributed by Pablo Galindo and Mark Shannon in :issue:`45256`.) @@ -1415,7 +1422,7 @@ We measured a 1-3% improvement in pyperformance. PEP 659: Specializing Adaptive Interpreter ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:pep:`659` is one of the key parts of the faster CPython project. The general +:pep:`659` is one of the key parts of the Faster CPython project. The general idea is that while Python is a dynamic language, most code has regions where objects and types rarely change. This concept is known as *type stability*. @@ -1424,17 +1431,18 @@ in the executing code. Python will then replace the current operation with a more specialized one. This specialized operation uses fast paths available only to those use cases/types, which generally outperform their generic counterparts. This also brings in another concept called *inline caching*, where -Python caches the results of expensive operations directly in the bytecode. +Python caches the results of expensive operations directly in the +:term:`bytecode`. The specializer will also combine certain common instruction pairs into one -superinstruction. This reduces the overhead during execution. +superinstruction, reducing the overhead during execution. Python will only specialize when it sees code that is "hot" (executed multiple times). This prevents Python -from wasting time for run-once code. Python can also de-specialize when code is +from wasting time on run-once code. Python can also de-specialize when code is too dynamic or when the use changes. Specialization is attempted periodically, -and specialization attempts are not too expensive. This allows specialization -to adapt to new circumstances. +and specialization attempts are not too expensive, +allowing specialization to adapt to new circumstances. (PEP written by Mark Shannon, with ideas inspired by Stefan Brunthaler. See :pep:`659` for more information. Implementation by Mark Shannon and Brandt @@ -1447,32 +1455,32 @@ Bucher, with additional help from Irit Katriel and Dennis Sweeney.) | Operation | Form | Specialization | Operation speedup | Contributor(s) | | | | | (up to) | | +===============+====================+=======================================================+===================+===================+ -| Binary | ``x+x; x*x; x-x;`` | Binary add, multiply and subtract for common types | 10% | Mark Shannon, | -| operations | | such as ``int``, ``float``, and ``str`` take custom | | Dong-hee Na, | -| | | fast paths for their underlying types. | | Brandt Bucher, | +| Binary | ``x + x`` | Binary add, multiply and subtract for common types | 10% | Mark Shannon, | +| operations | | such as :class:`int`, :class:`float` and :class:`str` | | Dong-hee Na, | +| | ``x - x`` | take custom fast paths for their underlying types. | | Brandt Bucher, | | | | | | Dennis Sweeney | +| | ``x * x`` | | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Subscript | ``a[i]`` | Subscripting container types such as ``list``, | 10-25% | Irit Katriel, | -| | | ``tuple`` and ``dict`` directly index the underlying | | Mark Shannon | -| | | data structures. | | | +| Subscript | ``a[i]`` | Subscripting container types such as :class:`list`, | 10-25% | Irit Katriel, | +| | | :class:`tuple` and :class:`dict` directly index | | Mark Shannon | +| | | the underlying data structures. | | | | | | | | | -| | | Subscripting custom ``__getitem__`` | | | +| | | Subscripting custom :meth:`~object.__getitem__` | | | | | | is also inlined similar to :ref:`inline-calls`. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ | Store | ``a[i] = z`` | Similar to subscripting specialization above. | 10-25% | Dennis Sweeney | | subscript | | | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ | Calls | ``f(arg)`` | Calls to common builtin (C) functions and types such | 20% | Mark Shannon, | -| | ``C(arg)`` | as ``len`` and ``str`` directly call their underlying | | Ken Jin | -| | | C version. This avoids going through the internal | | | -| | | calling convention. | | | -| | | | | | +| | | as :func:`len` and :class:`str` directly call their | | Ken Jin | +| | ``C(arg)`` | underlying C version. This avoids going through the | | | +| | | internal calling convention. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Load | ``print`` | The object's index in the globals/builtins namespace | [1]_ | Mark Shannon | -| global | ``len`` | is cached. Loading globals and builtins require | | | -| variable | | zero namespace lookups. | | | +| Load | ``print`` | The object's index in the globals/builtins namespace | [#load-global]_ | Mark Shannon | +| global | | is cached. Loading globals and builtins require | | | +| variable | ``len`` | zero namespace lookups. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Load | ``o.attr`` | Similar to loading global variables. The attribute's | [2]_ | Mark Shannon | +| Load | ``o.attr`` | Similar to loading global variables. The attribute's | [#load-attr]_ | Mark Shannon | | attribute | | index inside the class/object's namespace is cached. | | | | | | In most cases, attribute loading will require zero | | | | | | namespace lookups. | | | @@ -1484,14 +1492,15 @@ Bucher, with additional help from Irit Katriel and Dennis Sweeney.) | Store | ``o.attr = z`` | Similar to load attribute optimization. | 2% | Mark Shannon | | attribute | | | in pyperformance | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Unpack | ``*seq`` | Specialized for common containers such as ``list`` | 8% | Brandt Bucher | -| Sequence | | and ``tuple``. Avoids internal calling convention. | | | +| Unpack | ``*seq`` | Specialized for common containers such as | 8% | Brandt Bucher | +| Sequence | | :class:`list` and :class:`tuple`. | | | +| | | Avoids internal calling convention. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -.. [1] A similar optimization already existed since Python 3.8. 3.11 - specializes for more forms and reduces some overhead. +.. [#load-global] A similar optimization already existed since Python 3.8. + 3.11 specializes for more forms and reduces some overhead. -.. [2] A similar optimization already existed since Python 3.10. +.. [#load-attr] A similar optimization already existed since Python 3.10. 3.11 specializes for more forms. Furthermore, all attribute loads should be sped up by :issue:`45947`. @@ -1501,49 +1510,72 @@ Bucher, with additional help from Irit Katriel and Dennis Sweeney.) Misc ---- -* Objects now require less memory due to lazily created object namespaces. Their - namespace dictionaries now also share keys more freely. +* Objects now require less memory due to lazily created object namespaces. + Their namespace dictionaries now also share keys more freely. (Contributed Mark Shannon in :issue:`45340` and :issue:`40116`.) +* "Zero-cost" exceptions are implemented, eliminating the cost + of :keyword:`try` statements when no exception is raised. + (Contributed by Mark Shannon in :issue:`40222`.) + * A more concise representation of exceptions in the interpreter reduced the time required for catching an exception by about 10%. (Contributed by Irit Katriel in :issue:`45711`.) +* :mod:`re`'s regular expression matching engine has been partially refactored, + and now uses computed gotos (or "threaded code") on supported platforms. As a + result, Python 3.11 executes the `pyperformance regular expression benchmarks + `_ up to 10% + faster than Python 3.10. + (Contributed by Brandt Bucher in :gh:`91404`.) + .. _whatsnew311-faster-cpython-faq: FAQ --- -| Q: How should I write my code to utilize these speedups? -| -| A: You don't have to change your code. Write Pythonic code that follows common - best practices. The Faster CPython project optimizes for common code - patterns we observe. -| -| -| Q: Will CPython 3.11 use more memory? -| -| A: Maybe not. We don't expect memory use to exceed 20% more than 3.10. - This is offset by memory optimizations for frame objects and object - dictionaries as mentioned above. -| -| -| Q: I don't see any speedups in my workload. Why? -| -| A: Certain code won't have noticeable benefits. If your code spends most of - its time on I/O operations, or already does most of its - computation in a C extension library like numpy, there won't be significant - speedup. This project currently benefits pure-Python workloads the most. -| -| Furthermore, the pyperformance figures are a geometric mean. Even within the - pyperformance benchmarks, certain benchmarks have slowed down slightly, while - others have sped up by nearly 2x! -| -| -| Q: Is there a JIT compiler? -| -| A: No. We're still exploring other optimizations. +.. _faster-cpython-faq-my-code: + +How should I write my code to utilize these speedups? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Write Pythonic code that follows common best practices; +you don't have to change your code. +The Faster CPython project optimizes for common code patterns we observe. + + +.. _faster-cpython-faq-memory: + +Will CPython 3.11 use more memory? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Maybe not; we don't expect memory use to exceed 20% higher than 3.10. +This is offset by memory optimizations for frame objects and object +dictionaries as mentioned above. + + +.. _faster-cpython-ymmv: + +I don't see any speedups in my workload. Why? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Certain code won't have noticeable benefits. If your code spends most of +its time on I/O operations, or already does most of its +computation in a C extension library like NumPy, there won't be significant +speedups. This project currently benefits pure-Python workloads the most. + +Furthermore, the pyperformance figures are a geometric mean. Even within the +pyperformance benchmarks, certain benchmarks have slowed down slightly, while +others have sped up by nearly 2x! + + +.. _faster-cpython-jit: + +Is there a JIT compiler? +^^^^^^^^^^^^^^^^^^^^^^^^ + +No. We're still exploring other optimizations. .. _whatsnew311-faster-cpython-about: @@ -2082,30 +2114,22 @@ Build Changes and WASI contributed by Christian Heimes in :gh:`90473`; platforms promoted in :gh:`95085`) -* Building Python now requires: +* Building CPython now requires: - * A `C11 `_ compiler. + * A `C11 `_ compiler and standard library. `Optional C11 features `_ are not required. - (Contributed by Victor Stinner in :issue:`46656`.) + (Contributed by Victor Stinner in :issue:`46656`, + :issue:`45440` and :issue:`46640`.) * Support for `IEEE 754 `_ floating point numbers. (Contributed by Victor Stinner in :issue:`46917`.) - * Support for `floating point Not-a-Number (NaN) - `_, - as the :c:macro:`!Py_NO_NAN` macro has been removed. - (Contributed by Victor Stinner in :issue:`46656`.) - - * A `C99 `_ - ```` header file providing the - :c:func:`!copysign`, :c:func:`!hypot`, :c:func:`!isfinite`, - :c:func:`!isinf`, :c:func:`!isnan`, and :c:func:`!round` functions - (contributed by Victor Stinner in :issue:`45440`); - and a :c:data:`!NAN` constant or the :c:func:`!__builtin_nan` function - (Contributed by Victor Stinner in :issue:`46640`). +* The :c:macro:`!Py_NO_NAN` macro has been removed. + Since CPython now requires IEEE 754 floats, NaN values are always available. + (Contributed by Victor Stinner in :issue:`46656`.) * The :mod:`tkinter` package now requires `Tcl/Tk `_ version 8.5.12 or newer. diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 30c7c4e3bbd067..9957da1fc5f22a 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -86,6 +86,7 @@ struct _Py_interp_static_objects { // hamt_empty is here instead of global because of its weakreflist. _PyGC_Head_UNUSED _hamt_empty_gc_not_used; PyHamtObject hamt_empty; + PyBaseExceptionObject last_resort_memory_error; } singletons; }; diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index deac145fff7627..46a52740eb8a0c 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -21,6 +21,6 @@ typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); -extern instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; -extern instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; +extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; +extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index b54adf04761d4e..a2cc7c87c2f3e2 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -14,6 +14,9 @@ extern "C" { #include "pycore_obmalloc_init.h" +extern PyTypeObject _PyExc_MemoryError; + + /* The static initializers defined here should only be used in the runtime init code (in pystate.c and pylifecycle.c). */ @@ -120,6 +123,9 @@ extern "C" { .ob_base = _PyObject_IMMORTAL_INIT(&_PyHamt_Type), \ .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \ }, \ + .last_resort_memory_error = { \ + _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError), \ + }, \ }, \ }, \ ._initial_thread = _PyThreadState_INIT, \ diff --git a/Lib/argparse.py b/Lib/argparse.py index 240625ff01084e..a819d2650e85f0 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -403,10 +403,18 @@ def _format_actions_usage(self, actions, groups): except ValueError: continue else: - end = start + len(group._group_actions) + group_action_count = len(group._group_actions) + end = start + group_action_count if actions[start:end] == group._group_actions: + + suppressed_actions_count = 0 for action in group._group_actions: group_actions.add(action) + if action.help is SUPPRESS: + suppressed_actions_count += 1 + + exposed_actions_count = group_action_count - suppressed_actions_count + if not group.required: if start in inserts: inserts[start] += ' [' @@ -416,7 +424,7 @@ def _format_actions_usage(self, actions, groups): inserts[end] += ']' else: inserts[end] = ']' - else: + elif exposed_actions_count > 1: if start in inserts: inserts[start] += ' (' else: @@ -490,7 +498,6 @@ def _format_actions_usage(self, actions, groups): text = _re.sub(r'(%s) ' % open, r'\1', text) text = _re.sub(r' (%s)' % close, r'\1', text) text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = _re.sub(r'\(([^|]*)\)', r'\1', text) text = text.strip() # return the text diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index a2e06d5ef72f42..1c20754b839b69 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -630,10 +630,6 @@ def ensure_future(coro_or_future, *, loop=None): If the argument is a Future, it is returned directly. """ - return _ensure_future(coro_or_future, loop=loop) - - -def _ensure_future(coro_or_future, *, loop=None): if futures.isfuture(coro_or_future): if loop is not None and loop is not futures._get_loop(coro_or_future): raise ValueError('The future belongs to a different loop than ' @@ -798,7 +794,7 @@ def _done_callback(fut): outer = None # bpo-46672 for arg in coros_or_futures: if arg not in arg_to_fut: - fut = _ensure_future(arg, loop=loop) + fut = ensure_future(arg, loop=loop) if loop is None: loop = futures._get_loop(fut) if fut is not arg: @@ -855,7 +851,7 @@ def shield(arg): weak references to tasks. A task that isn't referenced elsewhere may get garbage collected at any time, even before it's done. """ - inner = _ensure_future(arg) + inner = ensure_future(arg) if inner.done(): # Shortcut. return inner diff --git a/Lib/pathlib.py b/Lib/pathlib.py index dde573592fddce..d375529ff5f767 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -275,9 +275,20 @@ def __reduce__(self): def _parse_parts(cls, parts): if not parts: return '', '', [] + elif len(parts) == 1: + path = os.fspath(parts[0]) + else: + path = cls._flavour.join(*parts) sep = cls._flavour.sep altsep = cls._flavour.altsep - path = cls._flavour.join(*parts) + if isinstance(path, str): + # Force-cast str subclasses to str (issue #21127) + path = str(path) + else: + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") if altsep: path = path.replace(altsep, sep) drv, root, rel = cls._flavour.splitroot(path) @@ -288,32 +299,10 @@ def _parse_parts(cls, parts): parsed = [sys.intern(x) for x in unfiltered_parsed if x and x != '.'] return drv, root, parsed - @classmethod - def _parse_args(cls, args): - # This is useful when you don't want to create an instance, just - # canonicalize some constructor arguments. - parts = [] - for a in args: - if isinstance(a, PurePath): - parts += a._parts - else: - a = os.fspath(a) - if isinstance(a, str): - # Force-cast str subclasses to str (issue #21127) - parts.append(str(a)) - else: - raise TypeError( - "argument should be a str object or an os.PathLike " - "object returning str, not %r" - % type(a)) - return cls._parse_parts(parts) - @classmethod def _from_parts(cls, args): - # We need to call _parse_args on the instance, so as to get the - # right flavour. self = object.__new__(cls) - drv, root, parts = self._parse_args(args) + drv, root, parts = self._parse_parts(args) self._drv = drv self._root = root self._parts = parts @@ -572,7 +561,7 @@ def joinpath(self, *args): anchored). """ drv1, root1, parts1 = self._drv, self._root, self._parts - drv2, root2, parts2 = self._parse_args(args) + drv2, root2, parts2 = self._parse_parts(args) if root2: if not drv2 and drv1: return self._from_parsed_parts(drv1, root2, [drv1 + root2] + parts2[1:]) @@ -659,7 +648,7 @@ def match(self, path_pattern): return True # Can't subclass os.PathLike from PurePath and keep the constructor -# optimizations in PurePath._parse_args(). +# optimizations in PurePath.__slots__. os.PathLike.register(PurePath) @@ -704,11 +693,7 @@ def __new__(cls, *args, **kwargs): warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) if cls is Path: cls = WindowsPath if os.name == 'nt' else PosixPath - self = cls._from_parts(args) - if self._flavour is not os.path: - raise NotImplementedError("cannot instantiate %r on your system" - % (cls.__name__,)) - return self + return cls._from_parts(args) def _make_child_relpath(self, part): # This is an optimization used for dir walking. `part` must be @@ -1258,9 +1243,19 @@ class PosixPath(Path, PurePosixPath): """ __slots__ = () + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise NotImplementedError( + f"cannot instantiate {cls.__name__!r} on your system") + class WindowsPath(Path, PureWindowsPath): """Path subclass for Windows systems. On a Windows system, instantiating a Path should return this object. """ __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise NotImplementedError( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 30f3f673ada577..3292c30d5fb29b 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -21,6 +21,9 @@ Generate Plist example: + import datetime + import plistlib + pl = dict( aString = "Doodah", aList = ["A", "B", 12, 32.1, [1, 2, 3]], @@ -28,22 +31,28 @@ anInt = 728, aDict = dict( anotherString = "", - aUnicodeValue = "M\xe4ssig, Ma\xdf", + aThirdString = "M\xe4ssig, Ma\xdf", aTrueValue = True, aFalseValue = False, ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), + aDate = datetime.datetime.now() ) - with open(fileName, 'wb') as fp: - dump(pl, fp) + print(plistlib.dumps(pl).decode()) Parse Plist example: - with open(fileName, 'rb') as fp: - pl = load(fp) - print(pl["aKey"]) + import plistlib + + plist = b''' + + foo + bar + + ''' + pl = plistlib.loads(plist) + print(pl["foo"]) """ __all__ = [ "InvalidFileException", "FMT_XML", "FMT_BINARY", "load", "dump", "loads", "dumps", "UID" diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index a47f5a96b89722..9d2df59d36ff47 100755 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -35,6 +35,7 @@ def errprint(*args): sys.stderr.write(sep + str(arg)) sep = " " sys.stderr.write("\n") + sys.exit(1) def main(): import getopt @@ -44,7 +45,6 @@ def main(): opts, args = getopt.getopt(sys.argv[1:], "qv") except getopt.error as msg: errprint(msg) - return for o, a in opts: if o == '-q': filename_only = filename_only + 1 @@ -52,7 +52,6 @@ def main(): verbose = verbose + 1 if not args: errprint("Usage:", sys.argv[0], "[-v] file_or_directory ...") - return for arg in args: check(arg) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index cabb2f837693ff..861da2326d1214 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -3764,6 +3764,28 @@ class TestHelpUsage(HelpTestCase): version = '' +class TestHelpUsageWithParentheses(HelpTestCase): + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('positional', metavar='(example) positional'), + Sig('-p', '--optional', metavar='{1 (option A), 2 (option B)}'), + ] + + usage = '''\ + usage: PROG [-h] [-p {1 (option A), 2 (option B)}] (example) positional + ''' + help = usage + '''\ + + positional arguments: + (example) positional + + options: + -h, --help show this help message and exit + -p {1 (option A), 2 (option B)}, --optional {1 (option A), 2 (option B)} + ''' + version = '' + + class TestHelpOnlyUserGroups(HelpTestCase): """Test basic usage messages""" diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index ed80540b2b3852..d5c02ba4a01df9 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -159,7 +159,7 @@ async def test_wait_for_race_condition(self): fut = loop.create_future() task = asyncio.wait_for(fut, timeout=0.2) - loop.call_later(0.1, fut.set_result, "ok") + loop.call_soon(fut.set_result, "ok") res = await task self.assertEqual(res, "ok") diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 9e19af0ae90fc1..e7a79bc13b7f3d 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -926,6 +926,16 @@ def test_filter_pickle(self): f2 = filter(filter_char, "abcdeabcde") self.check_iter_pickle(f1, list(f2), proto) + def test_filter_dealloc(self): + # Tests recursive deallocation of nested filter objects using the + # thrashcan mechanism. See gh-102356 for more details. + max_iters = 1000000 + i = filter(bool, range(max_iters)) + for _ in range(max_iters): + i = filter(bool, i) + del i + gc.collect() + def test_getattr(self): self.assertTrue(getattr(sys, 'stdout') is sys.stdout) self.assertRaises(TypeError, getattr) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 4de91d52c6d10c..df9c1f6ba65deb 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -166,6 +166,33 @@ def test_constructor_common(self): self.assertEqual(P(P('a'), P('b')), P('a/b')) self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) + def test_bytes(self): + P = self.cls + message = (r"argument should be a str or an os\.PathLike object " + r"where __fspath__ returns a str, not 'bytes'") + with self.assertRaisesRegex(TypeError, message): + P(b'a') + with self.assertRaises(TypeError): + P(b'a', 'b') + with self.assertRaises(TypeError): + P('a', b'b') + with self.assertRaises(TypeError): + P('a').joinpath(b'b') + with self.assertRaises(TypeError): + P('a') / b'b' + with self.assertRaises(TypeError): + b'a' / P('b') + with self.assertRaises(TypeError): + P('a').match(b'b') + with self.assertRaises(TypeError): + P('a').relative_to(b'b') + with self.assertRaises(TypeError): + P('a').with_name(b'b') + with self.assertRaises(TypeError): + P('a').with_stem(b'b') + with self.assertRaises(TypeError): + P('a').with_suffix(b'b') + def _check_str_subclass(self, *args): # Issue #21127: it should be possible to construct a PurePath object # from a str subclass instance, and it then gets converted to diff --git a/Lib/test/test_pow.py b/Lib/test/test_pow.py index 5cea9ceb20f5cc..eeb482ec4b27e2 100644 --- a/Lib/test/test_pow.py +++ b/Lib/test/test_pow.py @@ -19,12 +19,11 @@ def powtest(self, type): self.assertEqual(pow(2, i), pow2) if i != 30 : pow2 = pow2*2 - for othertype in (int,): - for i in list(range(-10, 0)) + list(range(1, 10)): - ii = type(i) - for j in range(1, 11): - jj = -othertype(j) - pow(ii, jj) + for i in list(range(-10, 0)) + list(range(1, 10)): + ii = type(i) + inv = pow(ii, -1) # inverse of ii + for jj in range(-10, 0): + self.assertAlmostEqual(pow(ii, jj), pow(inv, -jj)) for othertype in int, float: for i in range(1, 100): diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index e0a82e95c486be..afb8da719b0eed 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -110,9 +110,10 @@ def test_errprint(self): for args, expected in tests: with self.subTest(arguments=args, expected=expected): - with captured_stderr() as stderr: - tabnanny.errprint(*args) - self.assertEqual(stderr.getvalue() , expected) + with self.assertRaises(SystemExit): + with captured_stderr() as stderr: + tabnanny.errprint(*args) + self.assertEqual(stderr.getvalue() , expected) class TestNannyNag(TestCase): @@ -203,14 +204,16 @@ def test_when_wrong_indented(self): err = ('unindent does not match any outer indentation level' ' (, line 3)\n') err = f"{file_path!r}: Indentation Error: {err}" - self.verify_tabnanny_check(file_path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(file_path, err=err) def test_when_tokenize_tokenerror(self): """A python source code file eligible for raising 'tokenize.TokenError'.""" with TemporaryPyFile(SOURCE_CODES["incomplete_expression"]) as file_path: err = "('EOF in multi-line statement', (7, 0))\n" err = f"{file_path!r}: Token Error: {err}" - self.verify_tabnanny_check(file_path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(file_path, err=err) def test_when_nannynag_error_verbose(self): """A python source code file eligible for raising `tabnanny.NannyNag`. @@ -236,7 +239,8 @@ def test_when_no_file(self): path = 'no_file.py' err = (f"{path!r}: I/O Error: [Errno {errno.ENOENT}] " f"{os.strerror(errno.ENOENT)}: {path!r}\n") - self.verify_tabnanny_check(path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(path, err=err) def test_errored_directory(self): """Directory containing wrongly indented python source code files.""" @@ -251,7 +255,8 @@ def test_errored_directory(self): err = ('unindent does not match any outer indentation level' ' (, line 3)\n') err = f"{e_file!r}: Indentation Error: {err}" - self.verify_tabnanny_check(tmp_dir, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(tmp_dir, err=err) class TestProcessTokens(TestCase): @@ -287,9 +292,12 @@ def test_with_errored_codes_samples(self): class TestCommandLine(TestCase): """Tests command line interface of `tabnanny`.""" - def validate_cmd(self, *args, stdout="", stderr="", partial=False): + def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failure=False): """Common function to assert the behaviour of command line interface.""" - _, out, err = script_helper.assert_python_ok('-m', 'tabnanny', *args) + if expect_failure: + _, out, err = script_helper.assert_python_failure('-m', 'tabnanny', *args) + else: + _, out, err = script_helper.assert_python_ok('-m', 'tabnanny', *args) # Note: The `splitlines()` will solve the problem of CRLF(\r) added # by OS Windows. out = os.fsdecode(out) @@ -310,7 +318,7 @@ def test_with_errored_file(self): stderr = f"{file_path!r}: Indentation Error: " stderr += ('unindent does not match any outer indentation level' ' (, line 3)') - self.validate_cmd(file_path, stderr=stderr) + self.validate_cmd(file_path, stderr=stderr, expect_failure=True) def test_with_error_free_file(self): """Should not display anything if python file is correctly indented.""" @@ -321,7 +329,7 @@ def test_command_usage(self): """Should display usage on no arguments.""" path = findfile('tabnanny.py') stderr = f"Usage: {path} [-v] file_or_directory ..." - self.validate_cmd(stderr=stderr) + self.validate_cmd(stderr=stderr, expect_failure=True) def test_quiet_flag(self): """Should display less when quite mode is on.""" diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 96496ec9279e3f..0483ca3aa42f94 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -473,7 +473,6 @@ def test_var_substitution(self): def test_bad_var_substitution(self): T = TypeVar('T') - P = ParamSpec("P") bad_args = ( (), (int, str), Union, Generic, Generic[T], Protocol, Protocol[T], @@ -1037,8 +1036,6 @@ class G2(Generic[Unpack[Ts]]): pass def test_repr_is_correct(self): Ts = TypeVarTuple('Ts') - T = TypeVar('T') - T2 = TypeVar('T2') class G1(Generic[*Ts]): pass class G2(Generic[Unpack[Ts]]): pass @@ -1307,7 +1304,7 @@ def test_callable_args_are_correct(self): i = Callable[[None], *Ts] j = Callable[[None], Unpack[Ts]] self.assertEqual(i.__args__, (type(None), *Ts)) - self.assertEqual(i.__args__, (type(None), Unpack[Ts])) + self.assertEqual(j.__args__, (type(None), Unpack[Ts])) k = Callable[[None], tuple[int, *Ts]] l = Callable[[None], Tuple[int, Unpack[Ts]]] @@ -1435,8 +1432,6 @@ def g(*args: *Ts): pass self.assertEqual(g.__annotations__, {'args': (*Ts,)[0]}) def test_variadic_args_with_ellipsis_annotations_are_correct(self): - Ts = TypeVarTuple('Ts') - def a(*args: *tuple[int, ...]): pass self.assertEqual(a.__annotations__, {'args': (*tuple[int, ...],)[0]}) @@ -2921,8 +2916,8 @@ class DI: def __init__(self): self.x = None - self.assertIsInstance(C(), P) - self.assertIsInstance(D(), P) + self.assertIsInstance(CI(), P) + self.assertIsInstance(DI(), P) def test_protocols_in_unions(self): class P(Protocol): @@ -4918,7 +4913,6 @@ def test_overload_registry_repeated(self): # Definitions needed for features introduced in Python 3.6 from test import ann_module, ann_module2, ann_module3, ann_module5, ann_module6 -import asyncio T_a = TypeVar('T_a') @@ -7077,16 +7071,6 @@ class C: self.assertEqual(get_type_hints(C, globals())['classvar'], ClassVar[int]) self.assertEqual(get_type_hints(C, globals())['const'], Final[int]) - def test_hash_eq(self): - self.assertEqual(len({Annotated[int, 4, 5], Annotated[int, 4, 5]}), 1) - self.assertNotEqual(Annotated[int, 4, 5], Annotated[int, 5, 4]) - self.assertNotEqual(Annotated[int, 4, 5], Annotated[str, 4, 5]) - self.assertNotEqual(Annotated[int, 4], Annotated[int, 4, 4]) - self.assertEqual( - {Annotated[int, 4, 5], Annotated[int, 4, 5], Annotated[T, 4, 5]}, - {Annotated[int, 4, 5], Annotated[T, 4, 5]} - ) - def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, "Cannot subclass .*Annotated"): class C(Annotated): @@ -7515,7 +7499,6 @@ class Y(Generic[P, T]): self.assertEqual(B.__args__, ((int, str,), Tuple[bytes, float])) def test_var_substitution(self): - T = TypeVar("T") P = ParamSpec("P") subst = P.__typing_subst__ self.assertEqual(subst((int, str)), (int, str)) @@ -7835,7 +7818,7 @@ def test_special_attrs2(self): self.assertEqual(fr.__module__, 'typing') # Forward refs are currently unpicklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(TypeError) as exc: + with self.assertRaises(TypeError): pickle.dumps(fr, proto) self.assertEqual(SpecialAttrsTests.TypeName.__name__, 'TypeName') diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index 80d4fbdd8e3606..b989284a640e14 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -6,7 +6,6 @@ import traceback import types import functools -import warnings from fnmatch import fnmatch, fnmatchcase diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 5ea2cfc5183ef5..63fa21b2b33d17 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -361,7 +361,7 @@ def library_recipes(): dict( name="SQLite 3.40.1", url="https://sqlite.org/2022/sqlite-autoconf-3400100.tar.gz", - checksum="5498af3a357753d473ee713e363fa5b7", + checksum="42175b1a1d23529cb133bbd2b5900afd", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' diff --git a/Misc/ACKS b/Misc/ACKS index c591cd3bfe4b9e..7bbde3af99782b 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -637,6 +637,7 @@ Tim Golden Yonatan Goldschmidt Mark Gollahon Mikhail Golubev +Marta Gómez Macías Guilherme Gonçalves Tiago Gonçalves Chris Gonnerman diff --git a/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst b/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst new file mode 100644 index 00000000000000..865cfde8b06359 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst @@ -0,0 +1,3 @@ +Explicitly mark C extension modules that need defined signed integer overflow, +and add a configure option :option:`--with-strict-overflow`. +Patch by Matthias Görgens and Shantanu Jain. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst new file mode 100644 index 00000000000000..c03fd5266bc301 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst @@ -0,0 +1,2 @@ +Fix a bug that caused a crash when deallocating deeply nested filter +objects. Patch by Marta Gómez Macías. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst new file mode 100644 index 00000000000000..9ffc67cfb7ed56 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst @@ -0,0 +1 @@ +Do not memoize incorrectly automatically generated loop rules in the parser. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst b/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst new file mode 100644 index 00000000000000..e00e71fb8554f3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst @@ -0,0 +1 @@ +Set exit code when :mod:`tabnanny` CLI exits on error. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst b/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst new file mode 100644 index 00000000000000..8421466cdbb3c9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst @@ -0,0 +1,2 @@ +Speed up :class:`pathlib.Path` construction by running the path flavour +compatibility check only when pathlib is imported. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst b/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst new file mode 100644 index 00000000000000..af4ee9ad904868 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst @@ -0,0 +1,2 @@ +Speed up :class:`pathlib.PurePath` construction by calling +:func:`os.path.join` only when two or more arguments are given. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst b/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst new file mode 100644 index 00000000000000..c05f92ae699de9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst @@ -0,0 +1,4 @@ +Speed up :class:`pathlib.PurePath` construction by handling arguments more +uniformly. When a :class:`pathlib.Path` argument is supplied, +we use its string representation rather than joining its parts +with :func:`os.path.join`. diff --git a/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst b/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst new file mode 100644 index 00000000000000..1efe72439b3a4a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst @@ -0,0 +1,2 @@ +Fix a bug where parentheses in the ``metavar`` argument to :meth:`argparse.ArgumentParser.add_argument` were +dropped. Patch by Yeojin Kim. diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 461c505c092c70..79dbe3474ba9e8 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -61,9 +61,9 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) static int _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) { - PyObject *exctype, *excval, *exctb; + PyObject *exc; if (ignoreexc) { - PyErr_Fetch(&exctype, &excval, &exctb); + exc = PyErr_GetRaisedException(); } int res = _PyCrossInterpreterData_Release(data); if (res < 0) { @@ -84,7 +84,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) } } if (ignoreexc) { - PyErr_Restore(exctype, excval, exctb); + PyErr_SetRaisedException(exc); } return res; } @@ -294,9 +294,9 @@ _sharedexception_free(_sharedexception *exc) } static _sharedexception * -_sharedexception_bind(PyObject *exctype, PyObject *exc, PyObject *tb) +_sharedexception_bind(PyObject *exc) { - assert(exctype != NULL); + assert(exc != NULL); char *failure = NULL; _sharedexception *err = _sharedexception_new(); @@ -304,7 +304,7 @@ _sharedexception_bind(PyObject *exctype, PyObject *exc, PyObject *tb) goto finally; } - PyObject *name = PyUnicode_FromFormat("%S", exctype); + PyObject *name = PyUnicode_FromFormat("%S", Py_TYPE(exc)); if (name == NULL) { failure = "unable to format exception type name"; goto finally; @@ -432,10 +432,7 @@ static int _run_script(PyInterpreterState *interp, const char *codestr, _sharedns *shared, _sharedexception **exc) { - PyObject *exctype = NULL; PyObject *excval = NULL; - PyObject *tb = NULL; - PyObject *main_mod = _PyInterpreterState_GetMainModule(interp); if (main_mod == NULL) { goto error; @@ -469,12 +466,9 @@ _run_script(PyInterpreterState *interp, const char *codestr, return 0; error: - PyErr_Fetch(&exctype, &excval, &tb); - - _sharedexception *sharedexc = _sharedexception_bind(exctype, excval, tb); - Py_XDECREF(exctype); + excval = PyErr_GetRaisedException(); + _sharedexception *sharedexc = _sharedexception_bind(excval); Py_XDECREF(excval); - Py_XDECREF(tb); if (sharedexc == NULL) { fprintf(stderr, "RunFailedError: script raised an uncaught exception"); PyErr_Clear(); diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 976f84dbf63c93..a473cbdfeda7fc 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3207,8 +3207,6 @@ SimpleExtendsException(PyExc_Exception, ReferenceError, #define MEMERRORS_SAVE 16 -static PyBaseExceptionObject last_resort_memory_error; - static PyObject * get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) { @@ -3216,7 +3214,9 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) struct _Py_exc_state *state = get_exc_state(); if (state->memerrors_freelist == NULL) { if (!allow_allocation) { - return Py_NewRef(&last_resort_memory_error); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return Py_NewRef( + &_Py_INTERP_SINGLETON(interp, last_resort_memory_error)); } PyObject *result = BaseException_new((PyTypeObject *)PyExc_MemoryError, args, kwds); return result; @@ -3239,8 +3239,6 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) return (PyObject *)self; } -static PyBaseExceptionObject last_resort_memory_error; - static PyObject * MemoryError_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -3325,7 +3323,7 @@ free_preallocated_memerrors(struct _Py_exc_state *state) } -static PyTypeObject _PyExc_MemoryError = { +PyTypeObject _PyExc_MemoryError = { PyVarObject_HEAD_INIT(NULL, 0) "MemoryError", sizeof(PyBaseExceptionObject), @@ -3339,9 +3337,6 @@ static PyTypeObject _PyExc_MemoryError = { }; PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError; -static PyBaseExceptionObject last_resort_memory_error = { - _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError) -}; /* * BufferError extends Exception diff --git a/Parser/parser.c b/Parser/parser.c index 845c1739d63a60..e0a88a9cc72c8b 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -23777,7 +23777,6 @@ _loop0_1_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -23828,7 +23827,6 @@ _loop0_1_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_1_type, _seq); p->level--; return _seq; } @@ -23847,7 +23845,6 @@ _loop0_2_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -23898,7 +23895,6 @@ _loop0_2_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_2_type, _seq); p->level--; return _seq; } @@ -23917,7 +23913,6 @@ _loop1_3_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -23973,7 +23968,6 @@ _loop1_3_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_3_type, _seq); p->level--; return _seq; } @@ -23992,7 +23986,6 @@ _loop0_5_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -24052,7 +24045,6 @@ _loop0_5_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_5_type, _seq); p->level--; return _seq; } @@ -24585,7 +24577,6 @@ _loop1_14_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -24641,7 +24632,6 @@ _loop1_14_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_14_type, _seq); p->level--; return _seq; } @@ -24823,7 +24813,6 @@ _loop0_19_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -24883,7 +24872,6 @@ _loop0_19_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_19_type, _seq); p->level--; return _seq; } @@ -24944,7 +24932,6 @@ _loop0_21_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25004,7 +24991,6 @@ _loop0_21_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_21_type, _seq); p->level--; return _seq; } @@ -25170,7 +25156,6 @@ _loop0_24_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25221,7 +25206,6 @@ _loop0_24_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_24_type, _seq); p->level--; return _seq; } @@ -25240,7 +25224,6 @@ _loop1_25_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25296,7 +25279,6 @@ _loop1_25_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_25_type, _seq); p->level--; return _seq; } @@ -25315,7 +25297,6 @@ _loop0_27_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25375,7 +25356,6 @@ _loop0_27_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_27_type, _seq); p->level--; return _seq; } @@ -25483,7 +25463,6 @@ _loop0_30_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25543,7 +25522,6 @@ _loop0_30_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_30_type, _seq); p->level--; return _seq; } @@ -25651,7 +25629,6 @@ _loop1_32_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25707,7 +25684,6 @@ _loop1_32_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_32_type, _seq); p->level--; return _seq; } @@ -25870,7 +25846,6 @@ _loop0_36_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25921,7 +25896,6 @@ _loop0_36_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_36_type, _seq); p->level--; return _seq; } @@ -25940,7 +25914,6 @@ _loop0_37_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25991,7 +25964,6 @@ _loop0_37_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_37_type, _seq); p->level--; return _seq; } @@ -26010,7 +25982,6 @@ _loop0_38_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26061,7 +26032,6 @@ _loop0_38_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_38_type, _seq); p->level--; return _seq; } @@ -26080,7 +26050,6 @@ _loop1_39_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26136,7 +26105,6 @@ _loop1_39_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_39_type, _seq); p->level--; return _seq; } @@ -26155,7 +26123,6 @@ _loop0_40_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26206,7 +26173,6 @@ _loop0_40_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_40_type, _seq); p->level--; return _seq; } @@ -26225,7 +26191,6 @@ _loop1_41_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26281,7 +26246,6 @@ _loop1_41_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_41_type, _seq); p->level--; return _seq; } @@ -26300,7 +26264,6 @@ _loop1_42_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26356,7 +26319,6 @@ _loop1_42_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_42_type, _seq); p->level--; return _seq; } @@ -26375,7 +26337,6 @@ _loop1_43_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26431,7 +26392,6 @@ _loop1_43_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_43_type, _seq); p->level--; return _seq; } @@ -26450,7 +26410,6 @@ _loop0_44_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26501,7 +26460,6 @@ _loop0_44_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_44_type, _seq); p->level--; return _seq; } @@ -26520,7 +26478,6 @@ _loop1_45_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26576,7 +26533,6 @@ _loop1_45_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_45_type, _seq); p->level--; return _seq; } @@ -26595,7 +26551,6 @@ _loop0_46_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26646,7 +26601,6 @@ _loop0_46_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_46_type, _seq); p->level--; return _seq; } @@ -26665,7 +26619,6 @@ _loop1_47_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26721,7 +26674,6 @@ _loop1_47_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_47_type, _seq); p->level--; return _seq; } @@ -26740,7 +26692,6 @@ _loop0_48_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26791,7 +26742,6 @@ _loop0_48_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_48_type, _seq); p->level--; return _seq; } @@ -26810,7 +26760,6 @@ _loop0_49_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26861,7 +26810,6 @@ _loop0_49_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_49_type, _seq); p->level--; return _seq; } @@ -26880,7 +26828,6 @@ _loop1_50_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26936,7 +26883,6 @@ _loop1_50_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_50_type, _seq); p->level--; return _seq; } @@ -26955,7 +26901,6 @@ _loop0_52_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27015,7 +26960,6 @@ _loop0_52_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_52_type, _seq); p->level--; return _seq; } @@ -27076,7 +27020,6 @@ _loop0_54_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27136,7 +27079,6 @@ _loop0_54_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_54_type, _seq); p->level--; return _seq; } @@ -27197,7 +27139,6 @@ _loop0_56_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27257,7 +27198,6 @@ _loop0_56_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_56_type, _seq); p->level--; return _seq; } @@ -27318,7 +27258,6 @@ _loop0_58_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27378,7 +27317,6 @@ _loop0_58_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_58_type, _seq); p->level--; return _seq; } @@ -27516,7 +27454,6 @@ _loop1_60_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27572,7 +27509,6 @@ _loop1_60_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_60_type, _seq); p->level--; return _seq; } @@ -27591,7 +27527,6 @@ _loop1_61_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27647,7 +27582,6 @@ _loop1_61_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_61_type, _seq); p->level--; return _seq; } @@ -27760,7 +27694,6 @@ _loop1_64_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27816,7 +27749,6 @@ _loop1_64_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_64_type, _seq); p->level--; return _seq; } @@ -27835,7 +27767,6 @@ _loop0_66_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27895,7 +27826,6 @@ _loop0_66_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_66_type, _seq); p->level--; return _seq; } @@ -28226,7 +28156,6 @@ _loop0_72_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28286,7 +28215,6 @@ _loop0_72_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_72_type, _seq); p->level--; return _seq; } @@ -28347,7 +28275,6 @@ _loop0_74_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28407,7 +28334,6 @@ _loop0_74_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_74_type, _seq); p->level--; return _seq; } @@ -28526,7 +28452,6 @@ _loop0_77_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28586,7 +28511,6 @@ _loop0_77_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_77_type, _seq); p->level--; return _seq; } @@ -28647,7 +28571,6 @@ _loop0_79_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28707,7 +28630,6 @@ _loop0_79_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_79_type, _seq); p->level--; return _seq; } @@ -28768,7 +28690,6 @@ _loop1_80_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28824,7 +28745,6 @@ _loop1_80_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_80_type, _seq); p->level--; return _seq; } @@ -28843,7 +28763,6 @@ _loop1_81_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28899,7 +28818,6 @@ _loop1_81_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_81_type, _seq); p->level--; return _seq; } @@ -28918,7 +28836,6 @@ _loop0_83_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28978,7 +28895,6 @@ _loop0_83_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_83_type, _seq); p->level--; return _seq; } @@ -29039,7 +28955,6 @@ _loop1_84_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29095,7 +29010,6 @@ _loop1_84_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_84_type, _seq); p->level--; return _seq; } @@ -29114,7 +29028,6 @@ _loop1_85_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29170,7 +29083,6 @@ _loop1_85_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_85_type, _seq); p->level--; return _seq; } @@ -29189,7 +29101,6 @@ _loop1_86_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29245,7 +29156,6 @@ _loop1_86_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_86_type, _seq); p->level--; return _seq; } @@ -29308,7 +29218,6 @@ _loop0_89_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29368,7 +29277,6 @@ _loop0_89_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_89_type, _seq); p->level--; return _seq; } @@ -29765,7 +29673,6 @@ _loop0_95_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29816,7 +29723,6 @@ _loop0_95_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_95_type, _seq); p->level--; return _seq; } @@ -29835,7 +29741,6 @@ _loop0_96_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29886,7 +29791,6 @@ _loop0_96_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_96_type, _seq); p->level--; return _seq; } @@ -29905,7 +29809,6 @@ _loop0_97_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29956,7 +29859,6 @@ _loop0_97_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_97_type, _seq); p->level--; return _seq; } @@ -29975,7 +29877,6 @@ _loop1_98_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30031,7 +29932,6 @@ _loop1_98_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_98_type, _seq); p->level--; return _seq; } @@ -30050,7 +29950,6 @@ _loop0_99_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30101,7 +30000,6 @@ _loop0_99_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_99_type, _seq); p->level--; return _seq; } @@ -30120,7 +30018,6 @@ _loop1_100_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30176,7 +30073,6 @@ _loop1_100_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_100_type, _seq); p->level--; return _seq; } @@ -30195,7 +30091,6 @@ _loop1_101_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30251,7 +30146,6 @@ _loop1_101_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_101_type, _seq); p->level--; return _seq; } @@ -30270,7 +30164,6 @@ _loop1_102_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30326,7 +30219,6 @@ _loop1_102_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_102_type, _seq); p->level--; return _seq; } @@ -30345,7 +30237,6 @@ _loop0_103_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30396,7 +30287,6 @@ _loop0_103_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_103_type, _seq); p->level--; return _seq; } @@ -30415,7 +30305,6 @@ _loop1_104_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30471,7 +30360,6 @@ _loop1_104_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_104_type, _seq); p->level--; return _seq; } @@ -30490,7 +30378,6 @@ _loop0_105_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30541,7 +30428,6 @@ _loop0_105_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_105_type, _seq); p->level--; return _seq; } @@ -30560,7 +30446,6 @@ _loop1_106_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30616,7 +30501,6 @@ _loop1_106_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_106_type, _seq); p->level--; return _seq; } @@ -30635,7 +30519,6 @@ _loop0_107_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30686,7 +30569,6 @@ _loop0_107_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_107_type, _seq); p->level--; return _seq; } @@ -30705,7 +30587,6 @@ _loop1_108_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30761,7 +30642,6 @@ _loop1_108_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_108_type, _seq); p->level--; return _seq; } @@ -30780,7 +30660,6 @@ _loop1_109_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30836,7 +30715,6 @@ _loop1_109_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_109_type, _seq); p->level--; return _seq; } @@ -30905,7 +30783,6 @@ _loop0_112_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30965,7 +30842,6 @@ _loop0_112_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_112_type, _seq); p->level--; return _seq; } @@ -31026,7 +30902,6 @@ _loop1_113_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31082,7 +30957,6 @@ _loop1_113_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_113_type, _seq); p->level--; return _seq; } @@ -31101,7 +30975,6 @@ _loop0_114_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31152,7 +31025,6 @@ _loop0_114_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_114_type, _seq); p->level--; return _seq; } @@ -31171,7 +31043,6 @@ _loop0_115_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31222,7 +31093,6 @@ _loop0_115_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_115_type, _seq); p->level--; return _seq; } @@ -31301,7 +31171,6 @@ _loop0_118_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31361,7 +31230,6 @@ _loop0_118_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_118_type, _seq); p->level--; return _seq; } @@ -31470,7 +31338,6 @@ _loop0_121_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31530,7 +31397,6 @@ _loop0_121_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_121_type, _seq); p->level--; return _seq; } @@ -31591,7 +31457,6 @@ _loop0_123_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31651,7 +31516,6 @@ _loop0_123_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_123_type, _seq); p->level--; return _seq; } @@ -31712,7 +31576,6 @@ _loop0_125_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31772,7 +31635,6 @@ _loop0_125_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_125_type, _seq); p->level--; return _seq; } @@ -31833,7 +31695,6 @@ _loop0_127_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31893,7 +31754,6 @@ _loop0_127_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_127_type, _seq); p->level--; return _seq; } @@ -31954,7 +31814,6 @@ _loop0_128_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32005,7 +31864,6 @@ _loop0_128_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_128_type, _seq); p->level--; return _seq; } @@ -32024,7 +31882,6 @@ _loop0_130_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32084,7 +31941,6 @@ _loop0_130_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_130_type, _seq); p->level--; return _seq; } @@ -32145,7 +32001,6 @@ _loop1_131_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32201,7 +32056,6 @@ _loop1_131_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_131_type, _seq); p->level--; return _seq; } @@ -32261,7 +32115,6 @@ _loop0_134_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32321,7 +32174,6 @@ _loop0_134_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_134_type, _seq); p->level--; return _seq; } @@ -32382,7 +32234,6 @@ _loop0_136_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32442,7 +32293,6 @@ _loop0_136_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_136_type, _seq); p->level--; return _seq; } @@ -32503,7 +32353,6 @@ _loop0_138_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32563,7 +32412,6 @@ _loop0_138_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_138_type, _seq); p->level--; return _seq; } @@ -32624,7 +32472,6 @@ _loop0_140_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32684,7 +32531,6 @@ _loop0_140_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_140_type, _seq); p->level--; return _seq; } @@ -32745,7 +32591,6 @@ _loop0_142_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32805,7 +32650,6 @@ _loop0_142_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_142_type, _seq); p->level--; return _seq; } @@ -33557,7 +33401,6 @@ _loop0_154_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -33608,7 +33451,6 @@ _loop0_154_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_154_type, _seq); p->level--; return _seq; } @@ -33627,7 +33469,6 @@ _loop0_155_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -33678,7 +33519,6 @@ _loop0_155_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_155_type, _seq); p->level--; return _seq; } @@ -33697,7 +33537,6 @@ _loop0_156_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -33748,7 +33587,6 @@ _loop0_156_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_156_type, _seq); p->level--; return _seq; } @@ -34076,7 +33914,6 @@ _loop0_162_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34127,7 +33964,6 @@ _loop0_162_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_162_type, _seq); p->level--; return _seq; } @@ -34146,7 +33982,6 @@ _loop0_163_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34197,7 +34032,6 @@ _loop0_163_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_163_type, _seq); p->level--; return _seq; } @@ -34216,7 +34050,6 @@ _loop0_164_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34267,7 +34100,6 @@ _loop0_164_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_164_type, _seq); p->level--; return _seq; } @@ -34286,7 +34118,6 @@ _loop1_165_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34342,7 +34173,6 @@ _loop1_165_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_165_type, _seq); p->level--; return _seq; } @@ -34419,7 +34249,6 @@ _loop0_167_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34470,7 +34299,6 @@ _loop0_167_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_167_type, _seq); p->level--; return _seq; } @@ -34547,7 +34375,6 @@ _loop0_169_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34598,7 +34425,6 @@ _loop0_169_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_169_type, _seq); p->level--; return _seq; } @@ -34617,7 +34443,6 @@ _loop1_170_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34673,7 +34498,6 @@ _loop1_170_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_170_type, _seq); p->level--; return _seq; } @@ -34869,7 +34693,6 @@ _loop0_174_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34920,7 +34743,6 @@ _loop0_174_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_174_type, _seq); p->level--; return _seq; } @@ -35074,7 +34896,6 @@ _loop1_177_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35130,7 +34951,6 @@ _loop1_177_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_177_type, _seq); p->level--; return _seq; } @@ -35207,7 +35027,6 @@ _loop0_179_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35258,7 +35077,6 @@ _loop0_179_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_179_type, _seq); p->level--; return _seq; } @@ -35277,7 +35095,6 @@ _loop0_180_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35328,7 +35145,6 @@ _loop0_180_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_180_type, _seq); p->level--; return _seq; } @@ -35347,7 +35163,6 @@ _loop0_181_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35398,7 +35213,6 @@ _loop0_181_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_181_type, _seq); p->level--; return _seq; } @@ -35417,7 +35231,6 @@ _loop0_183_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35477,7 +35290,6 @@ _loop0_183_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_183_type, _seq); p->level--; return _seq; } @@ -35596,7 +35408,6 @@ _loop0_185_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35647,7 +35458,6 @@ _loop0_185_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_185_type, _seq); p->level--; return _seq; } @@ -35724,7 +35534,6 @@ _loop0_187_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35775,7 +35584,6 @@ _loop0_187_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_187_type, _seq); p->level--; return _seq; } @@ -35794,7 +35602,6 @@ _loop1_188_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35850,7 +35657,6 @@ _loop1_188_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_188_type, _seq); p->level--; return _seq; } @@ -35869,7 +35675,6 @@ _loop1_189_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35925,7 +35730,6 @@ _loop1_189_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_189_type, _seq); p->level--; return _seq; } @@ -36063,7 +35867,6 @@ _loop0_192_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36114,7 +35917,6 @@ _loop0_192_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_192_type, _seq); p->level--; return _seq; } @@ -36345,7 +36147,6 @@ _loop0_197_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36405,7 +36206,6 @@ _loop0_197_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_197_type, _seq); p->level--; return _seq; } @@ -36466,7 +36266,6 @@ _loop0_199_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36526,7 +36325,6 @@ _loop0_199_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_199_type, _seq); p->level--; return _seq; } @@ -36587,7 +36385,6 @@ _loop0_201_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36647,7 +36444,6 @@ _loop0_201_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_201_type, _seq); p->level--; return _seq; } @@ -36708,7 +36504,6 @@ _loop0_203_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36768,7 +36563,6 @@ _loop0_203_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_203_type, _seq); p->level--; return _seq; } @@ -36887,7 +36681,6 @@ _loop0_205_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36938,7 +36731,6 @@ _loop0_205_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_205_type, _seq); p->level--; return _seq; } @@ -36957,7 +36749,6 @@ _loop1_206_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37013,7 +36804,6 @@ _loop1_206_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_206_type, _seq); p->level--; return _seq; } @@ -37074,7 +36864,6 @@ _loop0_208_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37125,7 +36914,6 @@ _loop0_208_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_208_type, _seq); p->level--; return _seq; } @@ -37144,7 +36932,6 @@ _loop1_209_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37200,7 +36987,6 @@ _loop1_209_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_209_type, _seq); p->level--; return _seq; } @@ -37664,7 +37450,6 @@ _loop0_221_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37724,7 +37509,6 @@ _loop0_221_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_221_type, _seq); p->level--; return _seq; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 53439ab16040c4..12ca0ba6c4873c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -553,9 +553,11 @@ static void filter_dealloc(filterobject *lz) { PyObject_GC_UnTrack(lz); + Py_TRASHCAN_BEGIN(lz, filter_dealloc) Py_XDECREF(lz->func); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); + Py_TRASHCAN_END } static int diff --git a/Python/compile.c b/Python/compile.c index 2e60a8157533ad..45c97b4f8ef0e6 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -175,19 +175,18 @@ static struct jump_target_label_ NO_LABEL = {-1}; #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL))) #define NEW_JUMP_TARGET_LABEL(C, NAME) \ - jump_target_label NAME = cfg_new_label(CFG_BUILDER(C)); \ + jump_target_label NAME = instr_sequence_new_label(INSTR_SEQUENCE(C)); \ if (!IS_LABEL(NAME)) { \ return ERROR; \ } #define USE_LABEL(C, LBL) \ - RETURN_IF_ERROR(cfg_builder_use_label(CFG_BUILDER(C), LBL)) + RETURN_IF_ERROR(instr_sequence_use_label(INSTR_SEQUENCE(C), (LBL).id)) -struct instr { +struct cfg_instr { int i_opcode; int i_oparg; location i_loc; - /* The following fields should not be set by the front-end: */ struct basicblock_ *i_target; /* target block (if jump instruction) */ struct basicblock_ *i_except; /* target block when exception is raised */ }; @@ -196,7 +195,7 @@ struct instr { #define INSTR_SET_OP1(I, OP, ARG) \ do { \ assert(HAS_ARG(OP)); \ - struct instr *_instr__ptr_ = (I); \ + struct cfg_instr *_instr__ptr_ = (I); \ _instr__ptr_->i_opcode = (OP); \ _instr__ptr_->i_oparg = (ARG); \ } while (0); @@ -205,7 +204,7 @@ struct instr { #define INSTR_SET_OP0(I, OP) \ do { \ assert(!HAS_ARG(OP)); \ - struct instr *_instr__ptr_ = (I); \ + struct cfg_instr *_instr__ptr_ = (I); \ _instr__ptr_->i_opcode = (OP); \ _instr__ptr_->i_oparg = 0; \ } while (0); @@ -235,25 +234,25 @@ is_bit_set_in_table(const uint32_t *table, int bitindex) { } static inline int -is_relative_jump(struct instr *i) +is_relative_jump(struct cfg_instr *i) { return is_bit_set_in_table(_PyOpcode_RelativeJump, i->i_opcode); } static inline int -is_block_push(struct instr *i) +is_block_push(struct cfg_instr *i) { return IS_BLOCK_PUSH_OPCODE(i->i_opcode); } static inline int -is_jump(struct instr *i) +is_jump(struct cfg_instr *i) { return IS_JUMP_OPCODE(i->i_opcode); } static int -instr_size(struct instr *instruction) +instr_size(struct cfg_instr *instruction) { int opcode = instruction->i_opcode; assert(!IS_PSEUDO_OPCODE(opcode)); @@ -265,7 +264,7 @@ instr_size(struct instr *instruction) } static void -write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) +write_instr(_Py_CODEUNIT *codestr, struct cfg_instr *instruction, int ilen) { int opcode = instruction->i_opcode; assert(!IS_PSEUDO_OPCODE(opcode)); @@ -313,7 +312,7 @@ typedef struct basicblock_ { /* Exception stack at start of block, used by assembler to create the exception handling table */ ExceptStack *b_exceptstack; /* pointer to an array of instructions, initially NULL */ - struct instr *b_instr; + struct cfg_instr *b_instr; /* If b_next is non-NULL, it is a pointer to the next block reached by normal control flow. */ struct basicblock_ *b_next; @@ -342,7 +341,7 @@ typedef struct basicblock_ { } basicblock; -static struct instr * +static struct cfg_instr * basicblock_last_instr(const basicblock *b) { assert(b->b_iused >= 0); if (b->b_iused > 0) { @@ -352,21 +351,15 @@ basicblock_last_instr(const basicblock *b) { return NULL; } -static inline int -basicblock_returns(const basicblock *b) { - struct instr *last = basicblock_last_instr(b); - return last && (last->i_opcode == RETURN_VALUE || last->i_opcode == RETURN_CONST); -} - static inline int basicblock_exits_scope(const basicblock *b) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); return last && IS_SCOPE_EXIT_OPCODE(last->i_opcode); } static inline int basicblock_nofallthrough(const basicblock *b) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); return (last && (IS_SCOPE_EXIT_OPCODE(last->i_opcode) || IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode))); @@ -415,10 +408,208 @@ typedef struct cfg_builder_ { basicblock *g_curblock; /* label for the next instruction to be placed */ jump_target_label g_current_label; - /* next free label id */ - int g_next_free_label; } cfg_builder; +typedef struct { + int i_opcode; + int i_oparg; + location i_loc; +} instruction; + + +typedef struct instr_sequence_ { + instruction *s_instrs; + int s_allocated; + int s_used; + + int *s_labelmap; /* label id --> instr offset */ + int s_labelmap_size; + int s_next_free_label; /* next free label id */ +} instr_sequence; + +#define INITIAL_INSTR_SEQUENCE_SIZE 100 +#define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 + +/* + * Resize the array if index is out of range. + * + * idx: the index we want to access + * arr: pointer to the array + * alloc: pointer to the capacity of the array + * default_alloc: initial number of items + * item_size: size of each item + * + */ +static int +ensure_array_large_enough(int idx, void **arr_, int *alloc, int default_alloc, size_t item_size) +{ + void *arr = *arr_; + if (arr == NULL) { + int new_alloc = default_alloc; + if (idx >= new_alloc) { + new_alloc = idx + default_alloc; + } + arr = PyObject_Calloc(new_alloc, item_size); + if (arr == NULL) { + PyErr_NoMemory(); + return ERROR; + } + *alloc = new_alloc; + } + else if (idx >= *alloc) { + size_t oldsize = *alloc * item_size; + int new_alloc = *alloc << 1; + if (idx >= new_alloc) { + new_alloc = idx + default_alloc; + } + size_t newsize = new_alloc * item_size; + + if (oldsize > (SIZE_MAX >> 1)) { + PyErr_NoMemory(); + return ERROR; + } + + assert(newsize > 0); + void *tmp = PyObject_Realloc(arr, newsize); + if (tmp == NULL) { + PyErr_NoMemory(); + return ERROR; + } + *alloc = new_alloc; + arr = tmp; + memset((char *)arr + oldsize, 0, newsize - oldsize); + } + + *arr_ = arr; + return SUCCESS; +} + +static int +instr_sequence_next_inst(instr_sequence *seq) { + assert(seq->s_instrs != NULL || seq->s_used == 0); + + RETURN_IF_ERROR( + ensure_array_large_enough(seq->s_used + 1, + (void**)&seq->s_instrs, + &seq->s_allocated, + INITIAL_INSTR_SEQUENCE_SIZE, + sizeof(instruction))); + assert(seq->s_used < seq->s_allocated); + return seq->s_used++; +} + +static jump_target_label +instr_sequence_new_label(instr_sequence *seq) +{ + jump_target_label lbl = {seq->s_next_free_label++}; + return lbl; +} + +static int +instr_sequence_use_label(instr_sequence *seq, int lbl) { + int old_size = seq->s_labelmap_size; + RETURN_IF_ERROR( + ensure_array_large_enough(lbl, + (void**)&seq->s_labelmap, + &seq->s_labelmap_size, + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, + sizeof(int))); + + for(int i = old_size; i < seq->s_labelmap_size; i++) { + seq->s_labelmap[i] = -111; /* something weird, for debugging */ + } + seq->s_labelmap[lbl] = seq->s_used; /* label refers to the next instruction */ + return SUCCESS; +} + +static int +instr_sequence_addop(instr_sequence *seq, int opcode, int oparg, location loc) +{ + assert(IS_WITHIN_OPCODE_RANGE(opcode)); + assert(!IS_ASSEMBLER_OPCODE(opcode)); + assert(HAS_ARG(opcode) || HAS_TARGET(opcode) || oparg == 0); + assert(0 <= oparg && oparg < (1 << 30)); + + int idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(idx); + instruction *ci = &seq->s_instrs[idx]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + return SUCCESS; +} + +static int +instr_sequence_insert_instruction(instr_sequence *seq, int pos, + int opcode, int oparg, location loc) +{ + assert(pos >= 0 && pos <= seq->s_used); + int last_idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(last_idx); + for (int i=last_idx-1; i >= pos; i--) { + seq->s_instrs[i+1] = seq->s_instrs[i]; + } + instruction *ci = &seq->s_instrs[pos]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + + /* fix the labels map */ + for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { + if (seq->s_labelmap[lbl] >= pos) { + seq->s_labelmap[lbl]++; + } + } + return SUCCESS; +} + +static void +instr_sequence_fini(instr_sequence *seq) { + PyObject_Free(seq->s_labelmap); + seq->s_labelmap = NULL; + + PyObject_Free(seq->s_instrs); + seq->s_instrs = NULL; +} + +static int basicblock_addop(basicblock *b, int opcode, int oparg, location loc); +static int cfg_builder_maybe_start_new_block(cfg_builder *g); + +static int +cfg_builder_use_label(cfg_builder *g, jump_target_label lbl) +{ + g->g_current_label = lbl; + return cfg_builder_maybe_start_new_block(g); +} + +static int +cfg_builder_addop(cfg_builder *g, int opcode, int oparg, location loc) +{ + RETURN_IF_ERROR(cfg_builder_maybe_start_new_block(g)); + return basicblock_addop(g->g_curblock, opcode, oparg, loc); +} + +static int cfg_builder_init(cfg_builder *g); + +static int +instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { + memset(g, 0, sizeof(cfg_builder)); + RETURN_IF_ERROR(cfg_builder_init(g)); + /* Note: there can be more than one label for the same offset */ + for (int i = 0; i < seq->s_used; i++) { + for (int j=0; j < seq->s_labelmap_size; j++) { + if (seq->s_labelmap[j] == i) { + jump_target_label lbl = {j}; + RETURN_IF_ERROR(cfg_builder_use_label(g, lbl)); + } + } + instruction *instr = &seq->s_instrs[i]; + RETURN_IF_ERROR(cfg_builder_addop(g, instr->i_opcode, instr->i_oparg, instr->i_loc)); + } + return SUCCESS; +} + + /* The following items change on entry and exit of code blocks. They must be saved and restored when returning to a block. */ @@ -445,7 +636,7 @@ struct compiler_unit { Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */ Py_ssize_t u_kwonlyargcount; /* number of keyword only arguments for block */ - cfg_builder u_cfg_builder; /* The control flow graph */ + instr_sequence u_instr_sequence; /* codegen output */ int u_nfblocks; struct fblockinfo u_fblock[CO_MAXBLOCKS]; @@ -481,7 +672,7 @@ struct compiler { PyArena *c_arena; /* pointer to memory allocation arena */ }; -#define CFG_BUILDER(C) (&((C)->u->u_cfg_builder)) +#define INSTR_SEQUENCE(C) (&((C)->u->u_instr_sequence)) typedef struct { @@ -511,9 +702,7 @@ typedef struct { static int basicblock_next_instr(basicblock *); -static basicblock *cfg_builder_new_block(cfg_builder *g); -static int cfg_builder_maybe_start_new_block(cfg_builder *g); -static int cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc); +static int codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc); static void compiler_free(struct compiler *); static int compiler_error(struct compiler *, location loc, const char *, ...); @@ -744,7 +933,8 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) return dest; } -static void +#ifndef NDEBUG +static bool cfg_builder_check(cfg_builder *g) { for (basicblock *block = g->g_block_list; block != NULL; block = block->b_list) { @@ -759,7 +949,11 @@ cfg_builder_check(cfg_builder *g) assert (block->b_ialloc == 0); } } + return true; } +#endif + +static basicblock *cfg_builder_new_block(cfg_builder *g); static int cfg_builder_init(cfg_builder *g) @@ -777,7 +971,7 @@ cfg_builder_init(cfg_builder *g) static void cfg_builder_fini(cfg_builder* g) { - cfg_builder_check(g); + assert(cfg_builder_check(g)); basicblock *b = g->g_block_list; while (b != NULL) { if (b->b_instr) { @@ -792,7 +986,7 @@ cfg_builder_fini(cfg_builder* g) static void compiler_unit_free(struct compiler_unit *u) { - cfg_builder_fini(&u->u_cfg_builder); + instr_sequence_fini(&u->u_instr_sequence); Py_CLEAR(u->u_ste); Py_CLEAR(u->u_name); Py_CLEAR(u->u_qualname); @@ -878,13 +1072,6 @@ compiler_set_qualname(struct compiler *c) return SUCCESS; } -static jump_target_label -cfg_new_label(cfg_builder *g) -{ - jump_target_label lbl = {g->g_next_free_label++}; - return lbl; -} - /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -912,13 +1099,6 @@ cfg_builder_use_next_block(cfg_builder *g, basicblock *block) return block; } -static int -cfg_builder_use_label(cfg_builder *g, jump_target_label lbl) -{ - g->g_current_label = lbl; - return cfg_builder_maybe_start_new_block(g); -} - static inline int basicblock_append_instructions(basicblock *target, basicblock *source) { @@ -958,40 +1138,15 @@ static int basicblock_next_instr(basicblock *b) { assert(b != NULL); - if (b->b_instr == NULL) { - b->b_instr = (struct instr *)PyObject_Calloc( - DEFAULT_BLOCK_SIZE, sizeof(struct instr)); - if (b->b_instr == NULL) { - PyErr_NoMemory(); - return ERROR; - } - b->b_ialloc = DEFAULT_BLOCK_SIZE; - } - else if (b->b_iused == b->b_ialloc) { - struct instr *tmp; - size_t oldsize, newsize; - oldsize = b->b_ialloc * sizeof(struct instr); - newsize = oldsize << 1; - if (oldsize > (SIZE_MAX >> 1)) { - PyErr_NoMemory(); - return ERROR; - } + RETURN_IF_ERROR( + ensure_array_large_enough( + b->b_iused + 1, + (void**)&b->b_instr, + &b->b_ialloc, + DEFAULT_BLOCK_SIZE, + sizeof(struct cfg_instr))); - if (newsize == 0) { - PyErr_NoMemory(); - return ERROR; - } - b->b_ialloc <<= 1; - tmp = (struct instr *)PyObject_Realloc( - (void *)b->b_instr, newsize); - if (tmp == NULL) { - PyErr_NoMemory(); - return ERROR; - } - b->b_instr = tmp; - memset((char *)b->b_instr + oldsize, 0, newsize - oldsize); - } return b->b_iused++; } @@ -1100,7 +1255,7 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) if (off < 0) { return ERROR; } - struct instr *i = &b->b_instr[off]; + struct cfg_instr *i = &b->b_instr[off]; i->i_opcode = opcode; i->i_oparg = oparg; i->i_target = NULL; @@ -1115,7 +1270,7 @@ cfg_builder_current_block_is_terminated(cfg_builder *g) if (IS_LABEL(g->g_current_label)) { return true; } - struct instr *last = basicblock_last_instr(g->g_curblock); + struct cfg_instr *last = basicblock_last_instr(g->g_curblock); return last && IS_TERMINATOR_OPCODE(last->i_opcode); } @@ -1135,17 +1290,10 @@ cfg_builder_maybe_start_new_block(cfg_builder *g) } static int -cfg_builder_addop(cfg_builder *g, int opcode, int oparg, location loc) -{ - RETURN_IF_ERROR(cfg_builder_maybe_start_new_block(g)); - return basicblock_addop(g->g_curblock, opcode, oparg, loc); -} - -static int -cfg_builder_addop_noarg(cfg_builder *g, int opcode, location loc) +codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { assert(!HAS_ARG(opcode)); - return cfg_builder_addop(g, opcode, 0, loc); + return instr_sequence_addop(seq, opcode, 0, loc); } static Py_ssize_t @@ -1303,7 +1451,7 @@ compiler_addop_load_const(struct compiler *c, location loc, PyObject *o) if (arg < 0) { return ERROR; } - return cfg_builder_addop_i(CFG_BUILDER(c), LOAD_CONST, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), LOAD_CONST, arg, loc); } static int @@ -1314,7 +1462,7 @@ compiler_addop_o(struct compiler *c, location loc, if (arg < 0) { return ERROR; } - return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); } static int @@ -1340,12 +1488,12 @@ compiler_addop_name(struct compiler *c, location loc, arg <<= 1; arg |= 1; } - return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); } /* Add an opcode with an integer argument */ static int -cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc) +codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) { /* oparg value is unsigned, but a signed C int is usually used to store it in the C code (like Python/ceval.c). @@ -1356,23 +1504,23 @@ cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc) EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */ int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); - return cfg_builder_addop(g, opcode, oparg_, loc); + return instr_sequence_addop(seq, opcode, oparg_, loc); } static int -cfg_builder_addop_j(cfg_builder *g, location loc, +codegen_addop_j(instr_sequence *seq, location loc, int opcode, jump_target_label target) { assert(IS_LABEL(target)); assert(IS_JUMP_OPCODE(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); - return cfg_builder_addop(g, opcode, target.id, loc); + return instr_sequence_addop(seq, opcode, target.id, loc); } #define ADDOP(C, LOC, OP) \ - RETURN_IF_ERROR(cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC))) + RETURN_IF_ERROR(codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) #define ADDOP_IN_SCOPE(C, LOC, OP) { \ - if (cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC)) < 0) { \ + if (codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC)) < 0) { \ compiler_exit_scope(C); \ return ERROR; \ } \ @@ -1407,10 +1555,10 @@ cfg_builder_addop_j(cfg_builder *g, location loc, RETURN_IF_ERROR(compiler_addop_name((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) #define ADDOP_I(C, LOC, OP, O) \ - RETURN_IF_ERROR(cfg_builder_addop_i(CFG_BUILDER(C), (OP), (O), (LOC))) + RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) #define ADDOP_JUMP(C, LOC, OP, O) \ - RETURN_IF_ERROR(cfg_builder_addop_j(CFG_BUILDER(C), (LOC), (OP), (O))) + RETURN_IF_ERROR(codegen_addop_j(INSTR_SEQUENCE(C), (LOC), (OP), (O))) #define ADDOP_COMPARE(C, LOC, CMP) \ RETURN_IF_ERROR(compiler_addcompare((C), (LOC), (cmpop_ty)(CMP))) @@ -1546,9 +1694,6 @@ compiler_enter_scope(struct compiler *c, identifier name, c->c_nestlevel++; - cfg_builder *g = CFG_BUILDER(c); - RETURN_IF_ERROR(cfg_builder_init(g)); - if (u->u_scope_type == COMPILER_SCOPE_MODULE) { loc.lineno = 0; } @@ -1582,7 +1727,6 @@ compiler_exit_scope(struct compiler *c) _PyErr_WriteUnraisableMsg("on removing the last compiler " "stack item", NULL); } - cfg_builder_check(CFG_BUILDER(c)); } else { c->u = NULL; @@ -2321,7 +2465,7 @@ compiler_check_debug_args(struct compiler *c, arguments_ty args) } static inline int -insert_instruction(basicblock *block, int pos, struct instr *instr) { +insert_instruction(basicblock *block, int pos, struct cfg_instr *instr) { RETURN_IF_ERROR(basicblock_next_instr(block)); for (int i = block->b_iused - 1; i > pos; i--) { block->b_instr[i] = block->b_instr[i-1]; @@ -2336,14 +2480,10 @@ wrap_in_stopiteration_handler(struct compiler *c) NEW_JUMP_TARGET_LABEL(c, handler); /* Insert SETUP_CLEANUP at start */ - struct instr setup = { - .i_opcode = SETUP_CLEANUP, - .i_oparg = handler.id, - .i_loc = NO_LOCATION, - .i_target = NULL, - }; RETURN_IF_ERROR( - insert_instruction(c->u->u_cfg_builder.g_entryblock, 0, &setup)); + instr_sequence_insert_instruction( + INSTR_SEQUENCE(c), 0, + SETUP_CLEANUP, handler.id, NO_LOCATION)); ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); ADDOP(c, NO_LOCATION, RETURN_VALUE); @@ -4032,7 +4172,7 @@ compiler_nameop(struct compiler *c, location loc, if (op == LOAD_GLOBAL) { arg <<= 1; } - return cfg_builder_addop_i(CFG_BUILDER(c), op, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), op, arg, loc); } static int @@ -6051,7 +6191,7 @@ emit_and_reset_fail_pop(struct compiler *c, location loc, } while (--pc->fail_pop_size) { USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); - if (cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, loc) < 0) { + if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, loc) < 0) { pc->fail_pop_size = 0; PyObject_Free(pc->fail_pop); pc->fail_pop = NULL; @@ -6491,7 +6631,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) pc->fail_pop = NULL; pc->fail_pop_size = 0; pc->on_top = 0; - if (cfg_builder_addop_i(CFG_BUILDER(c), COPY, 1, LOC(alt)) < 0 || + if (codegen_addop_i(INSTR_SEQUENCE(c), COPY, 1, LOC(alt)) < 0 || compiler_pattern(c, alt, pc) < 0) { goto error; } @@ -6554,7 +6694,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) } } assert(control); - if (cfg_builder_addop_j(CFG_BUILDER(c), LOC(alt), JUMP, end) < 0 || + if (codegen_addop_j(INSTR_SEQUENCE(c), LOC(alt), JUMP, end) < 0 || emit_and_reset_fail_pop(c, LOC(alt), pc) < 0) { goto error; @@ -6566,7 +6706,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // Need to NULL this for the PyObject_Free call in the error block. old_pc.fail_pop = NULL; // No match. Pop the remaining copy of the subject and fail: - if (cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, LOC(p)) < 0 || + if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, LOC(p)) < 0 || jump_to_fail_pop(c, LOC(p), pc, JUMP) < 0) { goto error; } @@ -6884,7 +7024,7 @@ stackdepth(basicblock *entryblock, int code_flags) assert(depth >= 0); basicblock *next = b->b_next; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; int effect = stack_effect(instr->i_opcode, instr->i_oparg, 0); if (effect == PY_INVALID_STACK_EFFECT) { PyErr_Format(PyExc_SystemError, @@ -6973,7 +7113,7 @@ blocksize(basicblock *b) } static basicblock * -push_except_block(ExceptStack *stack, struct instr *setup) { +push_except_block(ExceptStack *stack, struct cfg_instr *setup) { assert(is_block_push(setup)); int opcode = setup->i_opcode; basicblock * target = setup->i_target; @@ -7045,7 +7185,7 @@ label_exception_targets(basicblock *entryblock) { b->b_exceptstack = NULL; handler = except_stack_top(except_stack); for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr)) { if (!instr->i_target->b_visited) { ExceptStack *copy = copy_except_stack(except_stack); @@ -7123,7 +7263,7 @@ mark_except_handlers(basicblock *entryblock) { #endif for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i=0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr)) { instr->i_target->b_except_handler = 1; } @@ -7152,7 +7292,7 @@ mark_warm(basicblock *entryblock) { next->b_visited = 1; } for (int i=0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_jump(instr) && !instr->i_target->b_visited) { *sp++ = instr->i_target; instr->i_target->b_visited = 1; @@ -7197,7 +7337,7 @@ mark_cold(basicblock *entryblock) { } } for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_jump(instr)) { assert(i == b->b_iused - 1); basicblock *target = b->b_instr[i].i_target; @@ -7238,7 +7378,7 @@ push_cold_blocks_to_end(cfg_builder *g, int code_flags) { b->b_next = explicit_jump; /* set target */ - struct instr *last = basicblock_last_instr(explicit_jump); + struct cfg_instr *last = basicblock_last_instr(explicit_jump); last->i_target = explicit_jump->b_next; } } @@ -7293,7 +7433,7 @@ static void convert_exception_handlers_to_nops(basicblock *entryblock) { for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr) || instr->i_opcode == POP_BLOCK) { INSTR_SET_OP0(instr, NOP); } @@ -7372,7 +7512,7 @@ assemble_exception_table(struct assembler *a, basicblock *entryblock) for (b = entryblock; b != NULL; b = b->b_next) { ioffset = b->b_offset; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (instr->i_except != handler) { if (handler != NULL) { RETURN_IF_ERROR( @@ -7542,7 +7682,7 @@ assemble_emit_location(struct assembler* a, location loc, int isize) */ static int -assemble_emit(struct assembler *a, struct instr *i) +assemble_emit(struct assembler *a, struct cfg_instr *i) { Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode); _Py_CODEUNIT *code; @@ -7562,7 +7702,7 @@ assemble_emit(struct assembler *a, struct instr *i) static int normalize_jumps_in_block(cfg_builder *g, basicblock *b) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last == NULL || !is_jump(last)) { return SUCCESS; } @@ -7663,7 +7803,7 @@ assemble_jump_offsets(basicblock *entryblock) for (basicblock *b = entryblock; b != NULL; b = b->b_next) { bsize = b->b_offset; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; int isize = instr_size(instr); /* Relative jumps are computed relative to the instruction pointer after fetching @@ -7735,7 +7875,7 @@ scan_block_for_locals(basicblock *b, basicblock ***sp) // bit i is set if local i is potentially uninitialized uint64_t unsafe_mask = b->b_unsafe_locals_mask; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); if (instr->i_except != NULL) { @@ -7768,7 +7908,7 @@ scan_block_for_locals(basicblock *b, basicblock ***sp) if (b->b_next && BB_HAS_FALLTHROUGH(b)) { maybe_push(b->b_next, unsafe_mask, sp); } - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last && is_jump(last)) { assert(last->i_target != NULL); maybe_push(last->i_target, unsafe_mask, sp); @@ -7791,7 +7931,7 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals) for (basicblock *b = entryblock; b != NULL; b = b->b_next) { blocknum++; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); int arg = instr->i_oparg; @@ -8035,7 +8175,6 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, PyObject *consts = NULL; PyObject *localsplusnames = NULL; PyObject *localspluskinds = NULL; - names = dict_keys_inorder(c->u->u_names, 0); if (!names) { goto error; @@ -8121,7 +8260,7 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, /* For debugging purposes only */ #if 0 static void -dump_instr(struct instr *i) +dump_instr(struct cfg_instr *i) { const char *jrel = (is_relative_jump(i)) ? "jrel " : ""; const char *jabs = (is_jump(i) && !is_relative_jump(i))? "jabs " : ""; @@ -8139,6 +8278,12 @@ dump_instr(struct instr *i) i->i_loc.lineno, i->i_opcode, arg, jabs, jrel); } +static inline int +basicblock_returns(const basicblock *b) { + struct cfg_instr *last = basicblock_last_instr(b); + return last && (last->i_opcode == RETURN_VALUE || last->i_opcode == RETURN_CONST); +} + static void dump_basicblock(const basicblock *b) { @@ -8211,14 +8356,14 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock, /* Add the generator prefix instructions. */ if (code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { - struct instr make_gen = { + struct cfg_instr make_gen = { .i_opcode = RETURN_GENERATOR, .i_oparg = 0, .i_loc = LOCATION(c->u->u_firstlineno, c->u->u_firstlineno, -1, -1), .i_target = NULL, }; RETURN_IF_ERROR(insert_instruction(entryblock, 0, &make_gen)); - struct instr pop_top = { + struct cfg_instr pop_top = { .i_opcode = POP_TOP, .i_oparg = 0, .i_loc = NO_LOCATION, @@ -8247,7 +8392,7 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock, if (oldindex == -1) { continue; } - struct instr make_cell = { + struct cfg_instr make_cell = { .i_opcode = MAKE_CELL, // This will get fixed in offset_derefs(). .i_oparg = oldindex, @@ -8261,7 +8406,7 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock, } if (nfreevars) { - struct instr copy_frees = { + struct cfg_instr copy_frees = { .i_opcode = COPY_FREE_VARS, .i_oparg = nfreevars, .i_loc = NO_LOCATION, @@ -8282,7 +8427,7 @@ guarantee_lineno_for_exits(basicblock *entryblock, int firstlineno) { int lineno = firstlineno; assert(lineno > 0); for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last == NULL) { continue; } @@ -8324,7 +8469,7 @@ fix_cell_offsets(struct compiler *c, basicblock *entryblock, int *fixedmap) // Then update offsets, either relative to locals or by cell2arg. for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *inst = &b->b_instr[i]; + struct cfg_instr *inst = &b->b_instr[i]; // This is called before extended args are generated. assert(inst->i_opcode != EXTENDED_ARG); int oldoffset = inst->i_oparg; @@ -8364,7 +8509,7 @@ no_redundant_nops(cfg_builder *g) { static bool no_redundant_jumps(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last != NULL) { if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { assert(last->i_target != b->b_next); @@ -8382,7 +8527,7 @@ opcode_metadata_is_sane(cfg_builder *g) { bool result = true; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; int opcode = instr->i_opcode; int oparg = instr->i_oparg; assert(opcode <= MAX_REAL_OPCODE); @@ -8426,7 +8571,7 @@ remove_redundant_jumps(cfg_builder *g) { */ assert(no_empty_basic_blocks(g)); for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); assert(last != NULL); assert(!IS_ASSEMBLER_OPCODE(last->i_opcode)); if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { @@ -8444,7 +8589,7 @@ remove_redundant_jumps(cfg_builder *g) { } static int -prepare_localsplus(struct compiler* c, int code_flags) +prepare_localsplus(struct compiler* c, cfg_builder *g, int code_flags) { assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX); assert(PyDict_GET_SIZE(c->u->u_cellvars) < INT_MAX); @@ -8460,7 +8605,6 @@ prepare_localsplus(struct compiler* c, int code_flags) return ERROR; } - cfg_builder* g = CFG_BUILDER(c); // This must be called before fix_cell_offsets(). if (insert_prefix_instructions(c, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { @@ -8474,20 +8618,21 @@ prepare_localsplus(struct compiler* c, int code_flags) if (numdropped < 0) { return ERROR; } + nlocalsplus -= numdropped; return nlocalsplus; } static int -add_return_at_end_of_block(struct compiler *c, int addNone) +add_return_at_end(struct compiler *c, int addNone) { - /* Make sure every block that falls off the end returns None. */ - if (!basicblock_returns(CFG_BUILDER(c)->g_curblock)) { - if (addNone) { - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - } - ADDOP(c, NO_LOCATION, RETURN_VALUE); + /* Make sure every instruction stream that falls off the end returns None. + * This also ensures that no jump target offsets are out of bounds. + */ + if (addNone) { + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); } + ADDOP(c, NO_LOCATION, RETURN_VALUE); return SUCCESS; } @@ -8496,6 +8641,8 @@ assemble(struct compiler *c, int addNone) { PyCodeObject *co = NULL; PyObject *consts = NULL; + cfg_builder g_; + cfg_builder *g = &g_; struct assembler a; memset(&a, 0, sizeof(struct assembler)); @@ -8504,12 +8651,18 @@ assemble(struct compiler *c, int addNone) return NULL; } - if (add_return_at_end_of_block(c, addNone) < 0) { + if (add_return_at_end(c, addNone) < 0) { return NULL; } + /** Preprocessing **/ + if (instr_sequence_to_cfg(INSTR_SEQUENCE(c), g) < 0) { + goto error; + } + assert(cfg_builder_check(g)); + int nblocks = 0; - for (basicblock *b = CFG_BUILDER(c)->g_block_list; b != NULL; b = b->b_list) { + for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) { nblocks++; } if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { @@ -8517,9 +8670,6 @@ assemble(struct compiler *c, int addNone) goto error; } - cfg_builder *g = CFG_BUILDER(c); - assert(g->g_entryblock != NULL); - /* Set firstlineno if it wasn't explicitly set. */ if (!c->u->u_firstlineno) { if (g->g_entryblock->b_instr && g->g_entryblock->b_instr->i_loc.lineno) { @@ -8530,9 +8680,8 @@ assemble(struct compiler *c, int addNone) } } - /** Preprocessing **/ /* Map labels to targets and mark exception handlers */ - if (translate_jump_labels_to_targets(g->g_entryblock)) { + if (translate_jump_labels_to_targets(g->g_entryblock) < 0) { goto error; } if (mark_except_handlers(g->g_entryblock) < 0) { @@ -8550,10 +8699,10 @@ assemble(struct compiler *c, int addNone) if (optimize_cfg(g, consts, c->c_const_cache)) { goto error; } - if (add_checks_for_loads_of_uninitialized_variables(g->g_entryblock, c) < 0) { + if (remove_unused_consts(g->g_entryblock, consts) < 0) { goto error; } - if (remove_unused_consts(g->g_entryblock, consts) < 0) { + if (add_checks_for_loads_of_uninitialized_variables(g->g_entryblock, c) < 0) { goto error; } @@ -8570,7 +8719,7 @@ assemble(struct compiler *c, int addNone) /** Assembly **/ - int nlocalsplus = prepare_localsplus(c, code_flags); + int nlocalsplus = prepare_localsplus(c, g, code_flags); if (nlocalsplus < 0) { goto error; } @@ -8587,7 +8736,6 @@ assemble(struct compiler *c, int addNone) if (normalize_jumps(g) < 0) { goto error; } - assert(no_redundant_jumps(g)); assert(opcode_metadata_is_sane(g)); @@ -8655,6 +8803,7 @@ assemble(struct compiler *c, int addNone) co = makecode(c, &a, consts, maxdepth, nlocalsplus, code_flags); error: Py_XDECREF(consts); + cfg_builder_fini(g); assemble_free(&a); return co; } @@ -8684,7 +8833,7 @@ get_const_value(int opcode, int oparg, PyObject *co_consts) */ static int fold_tuple_on_constants(PyObject *const_cache, - struct instr *inst, + struct cfg_instr *inst, int n, PyObject *consts) { /* Pre-conditions */ @@ -8753,7 +8902,7 @@ swaptimize(basicblock *block, int *ix) // NOTE: "./python -m test test_patma" serves as a good, quick stress test // for this function. Make sure to blow away cached *.pyc files first! assert(*ix < block->b_iused); - struct instr *instructions = &block->b_instr[*ix]; + struct cfg_instr *instructions = &block->b_instr[*ix]; // Find the length of the current sequence of SWAPs and NOPs, and record the // maximum depth of the stack manipulations: assert(instructions[0].i_opcode == SWAP); @@ -8854,7 +9003,7 @@ static int next_swappable_instruction(basicblock *block, int i, int lineno) { while (++i < block->b_iused) { - struct instr *instruction = &block->b_instr[i]; + struct cfg_instr *instruction = &block->b_instr[i]; if (0 <= lineno && instruction->i_loc.lineno != lineno) { // Optimizing across this instruction could cause user-visible // changes in the names bound between line tracing events! @@ -8880,7 +9029,7 @@ apply_static_swaps(basicblock *block, int i) // SWAPs are to our left, and potential swaperands are to our right: for (; 0 <= i; i--) { assert(i < block->b_iused); - struct instr *swap = &block->b_instr[i]; + struct cfg_instr *swap = &block->b_instr[i]; if (swap->i_opcode != SWAP) { if (swap->i_opcode == NOP || SWAPPABLE(swap->i_opcode)) { // Nope, but we know how to handle these. Keep looking: @@ -8903,7 +9052,7 @@ apply_static_swaps(basicblock *block, int i) } // Success! INSTR_SET_OP0(swap, NOP); - struct instr temp = block->b_instr[j]; + struct cfg_instr temp = block->b_instr[j]; block->b_instr[j] = block->b_instr[k]; block->b_instr[k] = temp; } @@ -8913,7 +9062,7 @@ apply_static_swaps(basicblock *block, int i) // target->i_target using the provided opcode. Return whether or not the // optimization was successful. static bool -jump_thread(struct instr *inst, struct instr *target, int opcode) +jump_thread(struct cfg_instr *inst, struct cfg_instr *target, int opcode) { assert(is_jump(inst)); assert(is_jump(target)); @@ -8938,11 +9087,11 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); - struct instr nop; + struct cfg_instr nop; INSTR_SET_OP0(&nop, NOP); - struct instr *target; + struct cfg_instr *target; for (int i = 0; i < bb->b_iused; i++) { - struct instr *inst = &bb->b_instr[i]; + struct cfg_instr *inst = &bb->b_instr[i]; int oparg = inst->i_oparg; int nextop = i+1 < bb->b_iused ? bb->b_instr[i+1].i_opcode : 0; if (HAS_TARGET(inst->i_opcode)) { @@ -9179,7 +9328,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) */ static int inline_small_exit_blocks(basicblock *bb) { - struct instr *last = basicblock_last_instr(bb); + struct cfg_instr *last = basicblock_last_instr(bb); if (last == NULL) { return 0; } @@ -9287,7 +9436,7 @@ mark_reachable(basicblock *entryblock) { } for (int i = 0; i < b->b_iused; i++) { basicblock *target; - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_jump(instr) || is_block_push(instr)) { target = instr->i_target; if (!target->b_visited) { @@ -9318,7 +9467,7 @@ eliminate_empty_basic_blocks(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { assert(b->b_iused > 0); for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (HAS_TARGET(instr->i_opcode)) { basicblock *target = instr->i_target; while (target->b_iused == 0) { @@ -9342,7 +9491,7 @@ eliminate_empty_basic_blocks(cfg_builder *g) { static void propagate_line_numbers(basicblock *entryblock) { for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last == NULL) { continue; } @@ -9398,7 +9547,7 @@ translate_jump_labels_to_targets(basicblock *entryblock) } for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; assert(instr->i_target == NULL); if (HAS_TARGET(instr->i_opcode)) { int lbl = instr->i_oparg; @@ -9584,7 +9733,7 @@ duplicate_exits_without_lineno(cfg_builder *g) */ basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); assert(last != NULL); if (is_jump(last)) { basicblock *target = last->i_target; @@ -9608,7 +9757,7 @@ duplicate_exits_without_lineno(cfg_builder *g) for (basicblock *b = entryblock; b != NULL; b = b->b_next) { if (BB_HAS_FALLTHROUGH(b) && b->b_next && b->b_iused > 0) { if (is_exit_without_lineno(b->b_next)) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); assert(last != NULL); b->b_next->b_instr[0].i_loc = last->i_loc; } @@ -9724,7 +9873,7 @@ cfg_to_instructions(cfg_builder *g) } for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; location loc = instr->i_loc; int arg = HAS_TARGET(instr->i_opcode) ? instr->i_target->b_label : instr->i_oparg; @@ -9783,16 +9932,19 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, goto finally; } - cfg_builder *g = CFG_BUILDER(c); - - if (translate_jump_labels_to_targets(g->g_entryblock) < 0) { + cfg_builder g; + if (instr_sequence_to_cfg(INSTR_SEQUENCE(c), &g) < 0) { + goto finally; + } + if (translate_jump_labels_to_targets(g.g_entryblock) < 0) { goto finally; } - res = cfg_to_instructions(g); + res = cfg_to_instructions(&g); finally: compiler_exit_scope(c); + cfg_builder_fini(&g); compiler_free(c); _PyArena_Free(arena); return res; @@ -9815,7 +9967,7 @@ _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts) if (const_cache == NULL) { goto error; } - if (translate_jump_labels_to_targets(g.g_entryblock)) { + if (translate_jump_labels_to_targets(g.g_entryblock) < 0) { goto error; } if (optimize_cfg(&g, consts, const_cache) < 0) { diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 9e90ef32130f1d..cca29d859902a4 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -199,7 +199,7 @@ list_to_tuple(PyThreadState* unused, PyObject *v) return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v)); } -instrinsic_func1 +const instrinsic_func1 _PyIntrinsics_UnaryFunctions[] = { [0] = no_intrinsic, [INTRINSIC_PRINT] = print_expr, @@ -221,7 +221,7 @@ prep_reraise_star(PyThreadState* unused, PyObject *orig, PyObject *excs) return _PyExc_PrepReraiseStar(orig, excs); } -instrinsic_func2 +const instrinsic_func2 _PyIntrinsics_BinaryFunctions[] = { [INTRINSIC_PREP_RERAISE_STAR] = prep_reraise_star, }; diff --git a/Tools/c-analyzer/c_parser/parser/_common.py b/Tools/c-analyzer/c_parser/parser/_common.py index d468d5442a939f..2eacace2c001df 100644 --- a/Tools/c-analyzer/c_parser/parser/_common.py +++ b/Tools/c-analyzer/c_parser/parser/_common.py @@ -7,13 +7,25 @@ ) -def log_match(group, m): +def log_match(group, m, depth_before=None, depth_after=None): from . import _logger - text = m.group(0) - if text.startswith(('(', ')')) or text.endswith(('(', ')')): - _logger.debug(f'matched <{group}> ({text!r})') + + if m is not None: + text = m.group(0) + if text.startswith(('(', ')')) or text.endswith(('(', ')')): + _logger.debug(f'matched <{group}> ({text!r})') + else: + _logger.debug(f'matched <{group}> ({text})') + + elif depth_before is not None or depth_after is not None: + if depth_before is None: + depth_before = '???' + elif depth_after is None: + depth_after = '???' + _logger.log(1, f'depth: %s -> %s', depth_before, depth_after) + else: - _logger.debug(f'matched <{group}> ({text})') + raise NotImplementedError('this should not have been hit') ############################# diff --git a/Tools/c-analyzer/c_parser/parser/_func_body.py b/Tools/c-analyzer/c_parser/parser/_func_body.py index 42fd459e111d2c..25f2f5807ae827 100644 --- a/Tools/c-analyzer/c_parser/parser/_func_body.py +++ b/Tools/c-analyzer/c_parser/parser/_func_body.py @@ -65,11 +65,11 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): ) = m.groups() if empty: - log_match('', m) + log_match('', m, depth) resolve(None, None, None, text) yield None, text elif inline_kind: - log_match('', m) + log_match('', m, depth) kind = inline_kind name = inline_name or anon_name('inline-') data = [] # members @@ -92,7 +92,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): # XXX Should "parent" really be None for inline type decls? yield resolve(kind, data, name, text, None), text elif block_close: - log_match('', m) + log_match('', m, depth) depth -= 1 resolve(None, None, None, text) # XXX This isn't great. Calling resolve() should have @@ -101,13 +101,13 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): # needs to be fixed. yield None, text elif compound_bare: - log_match('', m) + log_match('', m, depth) yield resolve('statement', compound_bare, None, text, parent), text elif compound_labeled: - log_match('', m) + log_match('', m, depth) yield resolve('statement', compound_labeled, None, text, parent), text elif compound_paren: - log_match('', m) + log_match('', m, depth) try: pos = match_paren(text) except ValueError: @@ -132,7 +132,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): } yield resolve('statement', data, None, text, parent), text elif block_open: - log_match('', m) + log_match('', m, depth) depth += 1 if block_leading: # An inline block: the last evaluated expression is used @@ -144,10 +144,10 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): resolve(None, None, None, text) yield None, text elif simple_ending: - log_match('', m) + log_match('', m, depth) yield resolve('statement', simple_stmt, None, text, parent), text elif var_ending: - log_match('', m) + log_match('', m, depth) kind = 'variable' _, name, vartype = parse_var_decl(decl) data = { @@ -220,7 +220,7 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth): remainder = srcinfo.text[m.end():] if inline_kind: - log_match('func inline', m) + log_match('func inline', m, depth, depth) kind = inline_kind name = inline_name or anon_name('inline-') # Immediately emit a forward declaration. @@ -249,7 +249,7 @@ def parse_body(source): yield parse_body, depth elif static_decl: - log_match('local variable', m) + log_match('local variable', m, depth, depth) _, name, data = parse_var_decl(static_decl) yield srcinfo.resolve('variable', data, name, parent=func), depth @@ -266,10 +266,13 @@ def parse_body(source): else: log_match('func other', m) if block_open: + log_match('func other', None, depth, depth + 1) depth += 1 elif block_close: + log_match('func other', None, depth, depth - 1) depth -= 1 elif stmt_end: + log_match('func other', None, depth, depth) pass else: # This should be unreachable. diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 770802253792d8..7ef1a8afc3b135 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -29,7 +29,7 @@ [^()]* )* ) # - ( [)] [)] )? # + ( [)] [)] ) # ''', re.VERBOSE) POST_ARGS = ( @@ -156,6 +156,7 @@ def _iter_top_include_lines(lines, topfile, cwd, if name != 'pragma': raise Exception(line) else: + line = re.sub(r'__inline__', 'inline', line) if not raw: line, partial = _strip_directives(line, partial=partial) yield _common.SourceLine( diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index ab1d6257f1b1a9..e7764165d36c4c 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -105,9 +105,14 @@ def clean_lines(text): * ./Include * ./Include/internal +Modules/_decimal/**/*.c Modules/_decimal/libmpdec +Modules/_hacl/*.c Modules/_hacl/include +Modules/_hacl/*.h Modules/_hacl/include Modules/_tkinter.c /usr/include/tcl8.6 +Modules/md5module.c Modules/_hacl/include +Modules/sha1module.c Modules/_hacl/include +Modules/sha2module.c Modules/_hacl/include Modules/tkappinit.c /usr/include/tcl -Modules/_decimal/**/*.c Modules/_decimal/libmpdec Objects/stringlib/*.h Objects # @end=tsv@ @@ -173,6 +178,7 @@ def clean_lines(text): Modules/_functoolsmodule.c Py_BUILD_CORE 1 Modules/_heapqmodule.c Py_BUILD_CORE 1 Modules/_io/*.c Py_BUILD_CORE 1 +Modules/_io/*.h Py_BUILD_CORE 1 Modules/_localemodule.c Py_BUILD_CORE 1 Modules/_operator.c Py_BUILD_CORE 1 Modules/_posixsubprocess.c Py_BUILD_CORE 1 @@ -296,6 +302,7 @@ def clean_lines(text): # First match wins. _abs('Modules/_ctypes/ctypes.h'): (5_000, 500), _abs('Modules/_datetimemodule.c'): (20_000, 300), + _abs('Modules/_hacl/*.c'): (200_000, 500), _abs('Modules/posixmodule.c'): (20_000, 500), _abs('Modules/termios.c'): (10_000, 800), _abs('Modules/_testcapimodule.c'): (20_000, 400), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 2e28c50c6ff69a..57b8542fb46482 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -539,7 +539,7 @@ Modules/_tkinter.c - command_mutex - Modules/_tkinter.c - HeadFHCD - Modules/_tkinter.c - stdin_ready - Modules/_tkinter.c - event_tstate - -Modules/_xxsubinterpretersmodule.c - _globals - +Modules/_xxinterpchannelsmodule.c - _globals - Modules/readline.c - completer_word_break_characters - Modules/readline.c - _history_length - Modules/readline.c - should_auto_add_history - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 849e20a1b0f4eb..700ddf2851839e 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -206,6 +206,10 @@ Modules/_decimal/_decimal.c - invalid_signals_err - Modules/_decimal/_decimal.c - signal_map - Modules/_decimal/_decimal.c - ssize_constants - Modules/_elementtree.c - ExpatMemoryHandler - +Modules/_hashopenssl.c - py_hashes - +Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - +Modules/_hacl/Hacl_Hash_MD5.c - _h0 - +Modules/_hacl/Hacl_Hash_MD5.c - _t - Modules/_io/_iomodule.c - static_types - Modules/_io/textio.c - encodefuncs - Modules/_io/winconsoleio.c - _PyWindowsConsoleIO_Type - @@ -218,9 +222,10 @@ Modules/_sre/sre_targets.h - sre_targets - Modules/_sre.c pattern_repr flag_names - Modules/_struct.c - bigendian_table - Modules/_struct.c - lilendian_table - +Modules/_struct.c - native_table - Modules/_tkinter.c - state_key - -Modules/_xxsubinterpretersmodule.c - _channelid_end_recv - -Modules/_xxsubinterpretersmodule.c - _channelid_end_send - +Modules/_xxinterpchannelsmodule.c - _channelid_end_recv - +Modules/_xxinterpchannelsmodule.c - _channelid_end_send - Modules/_zoneinfo.c - DAYS_BEFORE_MONTH - Modules/_zoneinfo.c - DAYS_IN_MONTH - Modules/arraymodule.c - descriptors - @@ -332,12 +337,15 @@ Python/frozen.c - _PyImport_FrozenTest - Python/getopt.c - longopts - Python/import.c - _PyImport_Inittab - Python/import.c - _PySys_ImplCacheTag - +Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - +Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - Python/pylifecycle.c - _TARGET_LOCALES - +Python/pylifecycle.c - INTERPRETER_TRAMPOLINE_CODEDEF - Python/pystate.c - initial - Python/specialize.c - adaptive_opcodes - Python/specialize.c - cache_requirements - @@ -392,8 +400,23 @@ Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - Modules/_testbuffer.c ndarray_push kwlist - Modules/_testbuffer.c staticarray_init kwlist - +Modules/_testcapi/code.c get_code_extra_index key - +Modules/_testcapi/datetime.c - test_run_counter - +Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - Modules/_testcapi/heaptype.c - _testcapimodule - +Modules/_testcapi/mem.c - FmData - +Modules/_testcapi/mem.c - FmHook - +Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - Modules/_testcapi/unicode.c - _testcapimodule - +Modules/_testcapi/watchers.c - g_dict_watch_events - +Modules/_testcapi/watchers.c - g_dict_watchers_installed - +Modules/_testcapi/watchers.c - g_type_modified_events - +Modules/_testcapi/watchers.c - g_type_watchers_installed - +Modules/_testcapi/watchers.c - num_code_object_created_events - +Modules/_testcapi/watchers.c - num_code_object_destroyed_events - +Modules/_testcapi/watchers.c - pyfunc_watchers - +Modules/_testcapi/watchers.c - func_watcher_ids - +Modules/_testcapi/watchers.c - func_watcher_callbacks - Modules/_testcapimodule.c - ContainerNoGC_members - Modules/_testcapimodule.c - ContainerNoGC_type - Modules/_testcapimodule.c - FmData - @@ -460,11 +483,13 @@ Modules/_testcapimodule.c - meth_static_methods - Modules/_testcapimodule.c - ml - Modules/_testcapimodule.c - str1 - Modules/_testcapimodule.c - str2 - +Modules/_testcapimodule.c - test_c_thread - Modules/_testcapimodule.c - test_members - Modules/_testcapimodule.c - test_run_counter - Modules/_testcapimodule.c - test_structmembersType - Modules/_testcapimodule.c - thread_done - Modules/_testcapimodule.c - x - +Modules/_testcapimodule.c - wait_done - Modules/_testcapimodule.c getargs_keyword_only keywords - Modules/_testcapimodule.c getargs_keywords keywords - Modules/_testcapimodule.c getargs_positional_only_and_keywords keywords - @@ -526,6 +551,7 @@ Modules/_testmultiphase.c - slots_exec_unreported_exception - Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots - Modules/_testmultiphase.c - testexport_methods - Modules/_testmultiphase.c - uninitialized_def - +Modules/_testsinglephase.c - global_state - Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - Modules/_xxtestfuzz/fuzzer.c - SRE_FLAG_DEBUG - diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index c41d87b74d1c49..e72ce7afdc4796 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -619,7 +619,8 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.add_return("_res") self.print("}") self.print("int _mark = p->mark;") - self.print("int _start_mark = p->mark;") + if memoize: + self.print("int _start_mark = p->mark;") self.print("void **_children = PyMem_Malloc(sizeof(void *));") self.out_of_memory_return(f"!_children") self.print("Py_ssize_t _children_capacity = 1;") @@ -642,7 +643,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.out_of_memory_return(f"!_seq", cleanup_code="PyMem_Free(_children);") self.print("for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") self.print("PyMem_Free(_children);") - if node.name: + if memoize and node.name: self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);") self.add_return("_seq") diff --git a/configure b/configure index 557519ad86e06d..9e99352f589f21 100755 --- a/configure +++ b/configure @@ -1054,6 +1054,7 @@ with_assertions enable_optimizations with_lto enable_bolt +with_strict_overflow with_dsymutil with_address_sanitizer with_memory_sanitizer @@ -1825,6 +1826,8 @@ Optional Packages: --with-lto=[full|thin|no|yes] enable Link-Time-Optimization in any build (default is no) + --with-strict-overflow if 'yes', add -fstrict-overflow to CFLAGS, else add + -fno-strict-overflow (default is no) --with-dsymutil link debug information into final executable with dsymutil in macOS (default is no) --with-address-sanitizer @@ -8376,6 +8379,64 @@ case $CC in fi esac +save_CFLAGS=$CFLAGS +CFLAGS="-fstrict-overflow -fno-strict-overflow" +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if $CC supports -fstrict-overflow and -fno-strict-overflow" >&5 +$as_echo_n "checking if $CC supports -fstrict-overflow and -fno-strict-overflow... " >&6; } +if ${ac_cv_cc_supports_fstrict_overflow+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_cv_cc_supports_fstrict_overflow=yes +else + ac_cv_cc_supports_fstrict_overflow=no + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_cc_supports_fstrict_overflow" >&5 +$as_echo "$ac_cv_cc_supports_fstrict_overflow" >&6; } +CFLAGS=$save_CFLAGS + +if test "x$ac_cv_cc_supports_fstrict_overflow" = xyes; then : + STRICT_OVERFLOW_CFLAGS="-fstrict-overflow" + NO_STRICT_OVERFLOW_CFLAGS="-fno-strict-overflow" +else + STRICT_OVERFLOW_CFLAGS="" + NO_STRICT_OVERFLOW_CFLAGS="" +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-strict-overflow" >&5 +$as_echo_n "checking for --with-strict-overflow... " >&6; } + +# Check whether --with-strict-overflow was given. +if test "${with_strict_overflow+set}" = set; then : + withval=$with_strict_overflow; + if test "x$ac_cv_cc_supports_fstrict_overflow" = xno; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: --with-strict-overflow=yes requires a compiler that supports -fstrict-overflow" >&5 +$as_echo "$as_me: WARNING: --with-strict-overflow=yes requires a compiler that supports -fstrict-overflow" >&2;} +fi + +else + with_strict_overflow=no + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_strict_overflow" >&5 +$as_echo "$with_strict_overflow" >&6; } + # Check if CC supports -Og optimization level save_CFLAGS=$CFLAGS CFLAGS="-Og" @@ -8428,15 +8489,8 @@ if test "${OPT-unset}" = "unset" then case $GCC in yes) - # For gcc 4.x we need to use -fwrapv so lets check if its supported - if "$CC" -v --help 2>/dev/null |grep -- -fwrapv > /dev/null; then - WRAP="-fwrapv" - fi - if test -n "${cc_is_clang}" then - # Clang also needs -fwrapv - WRAP="-fwrapv" # bpo-30104: disable strict aliasing to compile correctly dtoa.c, # see Makefile.pre.in for more information CFLAGS_ALIASING="-fno-strict-aliasing" @@ -8447,7 +8501,7 @@ then if test "$Py_DEBUG" = 'true' ; then OPT="-g $PYDEBUG_CFLAGS -Wall" else - OPT="-g $WRAP -O3 -Wall" + OPT="-g -O3 -Wall" fi ;; *) @@ -8580,6 +8634,12 @@ UNIVERSAL_ARCH_FLAGS= # tweak BASECFLAGS based on compiler and platform +if test "x$with_strict_overflow" = xyes; then : + BASECFLAGS="$BASECFLAGS $STRICT_OVERFLOW_CFLAGS" +else + BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS" +fi + case $GCC in yes) CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" @@ -27049,7 +27109,7 @@ fi as_fn_append MODULE_BLOCK "MODULE__CTYPES_STATE=$py_cv_module__ctypes$as_nl" if test "x$py_cv_module__ctypes" = xyes; then : - as_fn_append MODULE_BLOCK "MODULE__CTYPES_CFLAGS=$LIBFFI_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__CTYPES_CFLAGS=$NO_STRICT_OVERFLOW_CFLAGS $LIBFFI_CFLAGS$as_nl" as_fn_append MODULE_BLOCK "MODULE__CTYPES_LDFLAGS=$LIBFFI_LIBS$as_nl" fi diff --git a/configure.ac b/configure.ac index 982f669acbcfe5..31b7a2157a2bcc 100644 --- a/configure.ac +++ b/configure.ac @@ -2073,6 +2073,45 @@ case $CC in fi esac +dnl Historically, some of our code assumed that signed integer overflow +dnl is defined behaviour via twos-complement. +dnl Set STRICT_OVERFLOW_CFLAGS and NO_STRICT_OVERFLOW_CFLAGS depending on compiler support. +dnl Pass the latter to modules that depend on such behaviour. +_SAVE_VAR([CFLAGS]) +CFLAGS="-fstrict-overflow -fno-strict-overflow" +AC_CACHE_CHECK([if $CC supports -fstrict-overflow and -fno-strict-overflow], + [ac_cv_cc_supports_fstrict_overflow], + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[]], [[]])], + [ac_cv_cc_supports_fstrict_overflow=yes], + [ac_cv_cc_supports_fstrict_overflow=no] + ) +) +_RESTORE_VAR([CFLAGS]) + +AS_VAR_IF([ac_cv_cc_supports_fstrict_overflow], [yes], + [STRICT_OVERFLOW_CFLAGS="-fstrict-overflow" + NO_STRICT_OVERFLOW_CFLAGS="-fno-strict-overflow"], + [STRICT_OVERFLOW_CFLAGS="" + NO_STRICT_OVERFLOW_CFLAGS=""]) + +AC_MSG_CHECKING([for --with-strict-overflow]) +AC_ARG_WITH([strict-overflow], + AS_HELP_STRING( + [--with-strict-overflow], + [if 'yes', add -fstrict-overflow to CFLAGS, else add -fno-strict-overflow (default is no)] + ), + [ + AS_VAR_IF( + [ac_cv_cc_supports_fstrict_overflow], [no], + [AC_MSG_WARN([--with-strict-overflow=yes requires a compiler that supports -fstrict-overflow])], + [] + ) + ], + [with_strict_overflow=no] +) +AC_MSG_RESULT([$with_strict_overflow]) + # Check if CC supports -Og optimization level _SAVE_VAR([CFLAGS]) CFLAGS="-Og" @@ -2103,15 +2142,8 @@ if test "${OPT-unset}" = "unset" then case $GCC in yes) - # For gcc 4.x we need to use -fwrapv so lets check if its supported - if "$CC" -v --help 2>/dev/null |grep -- -fwrapv > /dev/null; then - WRAP="-fwrapv" - fi - if test -n "${cc_is_clang}" then - # Clang also needs -fwrapv - WRAP="-fwrapv" # bpo-30104: disable strict aliasing to compile correctly dtoa.c, # see Makefile.pre.in for more information CFLAGS_ALIASING="-fno-strict-aliasing" @@ -2122,7 +2154,7 @@ then if test "$Py_DEBUG" = 'true' ; then OPT="-g $PYDEBUG_CFLAGS -Wall" else - OPT="-g $WRAP -O3 -Wall" + OPT="-g -O3 -Wall" fi ;; *) @@ -2237,6 +2269,10 @@ AC_DEFUN([PY_CHECK_CC_WARNING], [ ]) # tweak BASECFLAGS based on compiler and platform +AS_VAR_IF([with_strict_overflow], [yes], + [BASECFLAGS="$BASECFLAGS $STRICT_OVERFLOW_CFLAGS"], + [BASECFLAGS="$BASECFLAGS $NO_STRICT_OVERFLOW_CFLAGS"]) + case $GCC in yes) CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" @@ -7213,7 +7249,7 @@ PY_STDLIB_MOD([_crypt], [$LIBCRYPT_CFLAGS], [$LIBCRYPT_LIBS]) PY_STDLIB_MOD([_ctypes], [], [test "$have_libffi" = yes], - [$LIBFFI_CFLAGS], [$LIBFFI_LIBS]) + [$NO_STRICT_OVERFLOW_CFLAGS $LIBFFI_CFLAGS], [$LIBFFI_LIBS]) PY_STDLIB_MOD([_curses], [], [test "$have_curses" != "no"], [$CURSES_CFLAGS], [$CURSES_LIBS]