diff --git a/.gitattributes b/.gitattributes index 5d5558da711b17..1f2f8a4cd8735a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -72,9 +72,11 @@ Doc/library/token-list.inc generated Include/internal/pycore_ast.h generated Include/internal/pycore_ast_state.h generated Include/internal/pycore_opcode.h generated +Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated Include/opcode.h generated Include/token.h generated +Lib/_opcode_metadata.py generated Lib/keyword.py generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 882ba9e9c9ebea..578cd71a7bd211 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -69,6 +69,7 @@ Python/traceback.c @iritkatriel # Import (including importlib). **/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw +/Python/import.c @kumaraditya303 **/*importlib/resources/* @jaraco @warsaw @FFY00 **/importlib/metadata/* @jaraco @warsaw diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index 737eb6dfebb77c..47037cd319e7e2 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -19,10 +19,8 @@ labels: "type-bug" -- [ ] I am confident this is a bug in CPython, - not a bug in a third-party project -- [ ] I have searched the CPython issue tracker, - and am confident this bug has not been reported before +- [ ] I am confident this is a bug in CPython, not a bug in a third-party project +- [ ] I have searched the CPython issue tracker, and am confident this bug has not been reported before ## A clear and concise description of the bug diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 60912a1ecaede2..dd53fe2bc95696 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1287,6 +1287,7 @@ function. You can create and destroy them using the following functions: in any thread where the sub-interpreter is currently active. Otherwise only multi-phase init extension modules (see :pep:`489`) may be imported. + (Also see :c:macro:`Py_mod_multiple_interpreters`.) This must be ``1`` (non-zero) if :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 8968b26b64320a..1df8c2b911ca8f 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -476,6 +476,10 @@ Customize Memory Allocators thread-safe: the :term:`GIL ` is not held when the allocator is called. + For the remaining domains, the allocator must also be thread-safe: + the allocator may be called in different interpreters that do not + share a ``GIL``. + If the new allocator is not a hook (does not call the previous allocator), the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the debug hooks on top on the new allocator. @@ -500,6 +504,8 @@ Customize Memory Allocators **must** wrap the existing allocator. Substituting the current allocator for some other arbitrary one is **not supported**. + .. versionchanged:: 3.12 + All allocators must be thread-safe. .. c:function:: void PyMem_SetupDebugHooks(void) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index fa5e97846e7bc3..e37505f6450a30 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -376,6 +376,37 @@ The available slot types are: If multiple ``Py_mod_exec`` slots are specified, they are processed in the order they appear in the *m_slots* array. +.. c:macro:: Py_mod_multiple_interpreters + + Specifies one of the following values: + + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED + + The module does not support being imported in subinterpreters. + + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED + + The module supports being imported in subinterpreters, + but only when they share the main interpreter's GIL. + (See :ref:`isolating-extensions-howto`.) + + .. c:macro:: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + + The module supports being imported in subinterpreters, + even when they have their own GIL. + (See :ref:`isolating-extensions-howto`.) + + This slot determines whether or not importing this module + in a subinterpreter will fail. + + Multiple ``Py_mod_multiple_interpreters`` slots may not be specified + in one module definition. + + If ``Py_mod_multiple_interpreters`` is not specified, the import + machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED``. + + .. versionadded:: 3.12 + See :PEP:`489` for more details on multi-phase initialization. Low-level module creation functions diff --git a/Doc/conf.py b/Doc/conf.py index 1b30b862f9a86c..49108eac58fc1d 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -110,6 +110,8 @@ ('c:type', 'uintptr_t'), ('c:type', 'va_list'), ('c:type', 'wchar_t'), + ('c:type', '__int64'), + ('c:type', 'unsigned __int64'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 472069032d6509..1134686c947d66 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -32,201 +32,201 @@ Annotations Best Practices Accessing The Annotations Dict Of An Object In Python 3.10 And Newer ==================================================================== - Python 3.10 adds a new function to the standard library: - :func:`inspect.get_annotations`. In Python versions 3.10 - and newer, calling this function is the best practice for - accessing the annotations dict of any object that supports - annotations. This function can also "un-stringize" - stringized annotations for you. - - If for some reason :func:`inspect.get_annotations` isn't - viable for your use case, you may access the - ``__annotations__`` data member manually. Best practice - for this changed in Python 3.10 as well: as of Python 3.10, - ``o.__annotations__`` is guaranteed to *always* work - on Python functions, classes, and modules. If you're - certain the object you're examining is one of these three - *specific* objects, you may simply use ``o.__annotations__`` - to get at the object's annotations dict. - - However, other types of callables--for example, - callables created by :func:`functools.partial`--may - not have an ``__annotations__`` attribute defined. When - accessing the ``__annotations__`` of a possibly unknown - object, best practice in Python versions 3.10 and - newer is to call :func:`getattr` with three arguments, - for example ``getattr(o, '__annotations__', None)``. - - Before Python 3.10, accessing ``__annotations__`` on a class that - defines no annotations but that has a parent class with - annotations would return the parent's ``__annotations__``. - In Python 3.10 and newer, the child class's annotations - will be an empty dict instead. +Python 3.10 adds a new function to the standard library: +:func:`inspect.get_annotations`. In Python versions 3.10 +and newer, calling this function is the best practice for +accessing the annotations dict of any object that supports +annotations. This function can also "un-stringize" +stringized annotations for you. + +If for some reason :func:`inspect.get_annotations` isn't +viable for your use case, you may access the +``__annotations__`` data member manually. Best practice +for this changed in Python 3.10 as well: as of Python 3.10, +``o.__annotations__`` is guaranteed to *always* work +on Python functions, classes, and modules. If you're +certain the object you're examining is one of these three +*specific* objects, you may simply use ``o.__annotations__`` +to get at the object's annotations dict. + +However, other types of callables--for example, +callables created by :func:`functools.partial`--may +not have an ``__annotations__`` attribute defined. When +accessing the ``__annotations__`` of a possibly unknown +object, best practice in Python versions 3.10 and +newer is to call :func:`getattr` with three arguments, +for example ``getattr(o, '__annotations__', None)``. + +Before Python 3.10, accessing ``__annotations__`` on a class that +defines no annotations but that has a parent class with +annotations would return the parent's ``__annotations__``. +In Python 3.10 and newer, the child class's annotations +will be an empty dict instead. Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== - In Python 3.9 and older, accessing the annotations dict - of an object is much more complicated than in newer versions. - The problem is a design flaw in these older versions of Python, - specifically to do with class annotations. +In Python 3.9 and older, accessing the annotations dict +of an object is much more complicated than in newer versions. +The problem is a design flaw in these older versions of Python, +specifically to do with class annotations. - Best practice for accessing the annotations dict of other - objects--functions, other callables, and modules--is the same - as best practice for 3.10, assuming you aren't calling - :func:`inspect.get_annotations`: you should use three-argument - :func:`getattr` to access the object's ``__annotations__`` - attribute. +Best practice for accessing the annotations dict of other +objects--functions, other callables, and modules--is the same +as best practice for 3.10, assuming you aren't calling +:func:`inspect.get_annotations`: you should use three-argument +:func:`getattr` to access the object's ``__annotations__`` +attribute. - Unfortunately, this isn't best practice for classes. The problem - is that, since ``__annotations__`` is optional on classes, and - because classes can inherit attributes from their base classes, - accessing the ``__annotations__`` attribute of a class may - inadvertently return the annotations dict of a *base class.* - As an example:: +Unfortunately, this isn't best practice for classes. The problem +is that, since ``__annotations__`` is optional on classes, and +because classes can inherit attributes from their base classes, +accessing the ``__annotations__`` attribute of a class may +inadvertently return the annotations dict of a *base class.* +As an example:: - class Base: - a: int = 3 - b: str = 'abc' + class Base: + a: int = 3 + b: str = 'abc' - class Derived(Base): - pass + class Derived(Base): + pass - print(Derived.__annotations__) + print(Derived.__annotations__) - This will print the annotations dict from ``Base``, not - ``Derived``. +This will print the annotations dict from ``Base``, not +``Derived``. - Your code will have to have a separate code path if the object - you're examining is a class (``isinstance(o, type)``). - In that case, best practice relies on an implementation detail - of Python 3.9 and before: if a class has annotations defined, - they are stored in the class's ``__dict__`` dictionary. Since - the class may or may not have annotations defined, best practice - is to call the ``get`` method on the class dict. +Your code will have to have a separate code path if the object +you're examining is a class (``isinstance(o, type)``). +In that case, best practice relies on an implementation detail +of Python 3.9 and before: if a class has annotations defined, +they are stored in the class's ``__dict__`` dictionary. Since +the class may or may not have annotations defined, best practice +is to call the ``get`` method on the class dict. - To put it all together, here is some sample code that safely - accesses the ``__annotations__`` attribute on an arbitrary - object in Python 3.9 and before:: +To put it all together, here is some sample code that safely +accesses the ``__annotations__`` attribute on an arbitrary +object in Python 3.9 and before:: - if isinstance(o, type): - ann = o.__dict__.get('__annotations__', None) - else: - ann = getattr(o, '__annotations__', None) + if isinstance(o, type): + ann = o.__dict__.get('__annotations__', None) + else: + ann = getattr(o, '__annotations__', None) - After running this code, ``ann`` should be either a - dictionary or ``None``. You're encouraged to double-check - the type of ``ann`` using :func:`isinstance` before further - examination. +After running this code, ``ann`` should be either a +dictionary or ``None``. You're encouraged to double-check +the type of ``ann`` using :func:`isinstance` before further +examination. - Note that some exotic or malformed type objects may not have - a ``__dict__`` attribute, so for extra safety you may also wish - to use :func:`getattr` to access ``__dict__``. +Note that some exotic or malformed type objects may not have +a ``__dict__`` attribute, so for extra safety you may also wish +to use :func:`getattr` to access ``__dict__``. Manually Un-Stringizing Stringized Annotations ============================================== - In situations where some annotations may be "stringized", - and you wish to evaluate those strings to produce the - Python values they represent, it really is best to - call :func:`inspect.get_annotations` to do this work - for you. - - If you're using Python 3.9 or older, or if for some reason - you can't use :func:`inspect.get_annotations`, you'll need - to duplicate its logic. You're encouraged to examine the - implementation of :func:`inspect.get_annotations` in the - current Python version and follow a similar approach. - - In a nutshell, if you wish to evaluate a stringized annotation - on an arbitrary object ``o``: - - * If ``o`` is a module, use ``o.__dict__`` as the - ``globals`` when calling :func:`eval`. - * If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` - as the ``globals``, and ``dict(vars(o))`` as the ``locals``, - when calling :func:`eval`. - * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, - :func:`functools.wraps`, or :func:`functools.partial`, iteratively - unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as - appropriate, until you have found the root unwrapped function. - * If ``o`` is a callable (but not a class), use - ``o.__globals__`` as the globals when calling :func:`eval`. - - However, not all string values used as annotations can - be successfully turned into Python values by :func:`eval`. - String values could theoretically contain any valid string, - and in practice there are valid use cases for type hints that - require annotating with string values that specifically - *can't* be evaluated. For example: - - * :pep:`604` union types using ``|``, before support for this - was added to Python 3.10. - * Definitions that aren't needed at runtime, only imported - when :const:`typing.TYPE_CHECKING` is true. - - If :func:`eval` attempts to evaluate such values, it will - fail and raise an exception. So, when designing a library - API that works with annotations, it's recommended to only - attempt to evaluate string values when explicitly requested - to by the caller. +In situations where some annotations may be "stringized", +and you wish to evaluate those strings to produce the +Python values they represent, it really is best to +call :func:`inspect.get_annotations` to do this work +for you. + +If you're using Python 3.9 or older, or if for some reason +you can't use :func:`inspect.get_annotations`, you'll need +to duplicate its logic. You're encouraged to examine the +implementation of :func:`inspect.get_annotations` in the +current Python version and follow a similar approach. + +In a nutshell, if you wish to evaluate a stringized annotation +on an arbitrary object ``o``: + +* If ``o`` is a module, use ``o.__dict__`` as the + ``globals`` when calling :func:`eval`. +* If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` + as the ``globals``, and ``dict(vars(o))`` as the ``locals``, + when calling :func:`eval`. +* If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, + :func:`functools.wraps`, or :func:`functools.partial`, iteratively + unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as + appropriate, until you have found the root unwrapped function. +* If ``o`` is a callable (but not a class), use + ``o.__globals__`` as the globals when calling :func:`eval`. + +However, not all string values used as annotations can +be successfully turned into Python values by :func:`eval`. +String values could theoretically contain any valid string, +and in practice there are valid use cases for type hints that +require annotating with string values that specifically +*can't* be evaluated. For example: + +* :pep:`604` union types using ``|``, before support for this + was added to Python 3.10. +* Definitions that aren't needed at runtime, only imported + when :const:`typing.TYPE_CHECKING` is true. + +If :func:`eval` attempts to evaluate such values, it will +fail and raise an exception. So, when designing a library +API that works with annotations, it's recommended to only +attempt to evaluate string values when explicitly requested +to by the caller. Best Practices For ``__annotations__`` In Any Python Version ============================================================ - * You should avoid assigning to the ``__annotations__`` member - of objects directly. Let Python manage setting ``__annotations__``. +* You should avoid assigning to the ``__annotations__`` member + of objects directly. Let Python manage setting ``__annotations__``. - * If you do assign directly to the ``__annotations__`` member - of an object, you should always set it to a ``dict`` object. +* If you do assign directly to the ``__annotations__`` member + of an object, you should always set it to a ``dict`` object. - * If you directly access the ``__annotations__`` member - of an object, you should ensure that it's a - dictionary before attempting to examine its contents. +* If you directly access the ``__annotations__`` member + of an object, you should ensure that it's a + dictionary before attempting to examine its contents. - * You should avoid modifying ``__annotations__`` dicts. +* You should avoid modifying ``__annotations__`` dicts. - * You should avoid deleting the ``__annotations__`` attribute - of an object. +* You should avoid deleting the ``__annotations__`` attribute + of an object. ``__annotations__`` Quirks ========================== - In all versions of Python 3, function - objects lazy-create an annotations dict if no annotations - are defined on that object. You can delete the ``__annotations__`` - attribute using ``del fn.__annotations__``, but if you then - access ``fn.__annotations__`` the object will create a new empty dict - that it will store and return as its annotations. Deleting the - annotations on a function before it has lazily created its annotations - dict will throw an ``AttributeError``; using ``del fn.__annotations__`` - twice in a row is guaranteed to always throw an ``AttributeError``. - - Everything in the above paragraph also applies to class and module - objects in Python 3.10 and newer. - - In all versions of Python 3, you can set ``__annotations__`` - on a function object to ``None``. However, subsequently - accessing the annotations on that object using ``fn.__annotations__`` - will lazy-create an empty dictionary as per the first paragraph of - this section. This is *not* true of modules and classes, in any Python - version; those objects permit setting ``__annotations__`` to any - Python value, and will retain whatever value is set. - - If Python stringizes your annotations for you - (using ``from __future__ import annotations``), and you - specify a string as an annotation, the string will - itself be quoted. In effect the annotation is quoted - *twice.* For example:: - - from __future__ import annotations - def foo(a: "str"): pass - - print(foo.__annotations__) - - This prints ``{'a': "'str'"}``. This shouldn't really be considered - a "quirk"; it's mentioned here simply because it might be surprising. +In all versions of Python 3, function +objects lazy-create an annotations dict if no annotations +are defined on that object. You can delete the ``__annotations__`` +attribute using ``del fn.__annotations__``, but if you then +access ``fn.__annotations__`` the object will create a new empty dict +that it will store and return as its annotations. Deleting the +annotations on a function before it has lazily created its annotations +dict will throw an ``AttributeError``; using ``del fn.__annotations__`` +twice in a row is guaranteed to always throw an ``AttributeError``. + +Everything in the above paragraph also applies to class and module +objects in Python 3.10 and newer. + +In all versions of Python 3, you can set ``__annotations__`` +on a function object to ``None``. However, subsequently +accessing the annotations on that object using ``fn.__annotations__`` +will lazy-create an empty dictionary as per the first paragraph of +this section. This is *not* true of modules and classes, in any Python +version; those objects permit setting ``__annotations__`` to any +Python value, and will retain whatever value is set. + +If Python stringizes your annotations for you +(using ``from __future__ import annotations``), and you +specify a string as an annotation, the string will +itself be quoted. In effect the annotation is quoted +*twice.* For example:: + + from __future__ import annotations + def foo(a: "str"): pass + + print(foo.__annotations__) + +This prints ``{'a': "'str'"}``. This shouldn't really be considered +a "quirk"; it's mentioned here simply because it might be surprising. diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 7aafd48711b58e..dcede13a03b4a5 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -13,8 +13,10 @@ Argument Clinic How-To .. topic:: Abstract Argument Clinic is a preprocessor for CPython C files. - Its purpose is to automate all the boilerplate involved - with writing argument parsing code for "builtins", + It was introduced in Python 3.4 with :pep:`436`, + in order to provide introspection signatures, + and to generate performant and tailor-made boilerplate code + for argument parsing in CPython builtins, module level functions, and class methods. This document is divided in four major sections: @@ -44,58 +46,6 @@ Argument Clinic How-To Background ========== - -The goals of Argument Clinic ----------------------------- - -Argument Clinic's primary goal -is to take over responsibility for all argument parsing code -inside CPython. This means that, when you convert a function -to work with Argument Clinic, that function should no longer -do any of its own argument parsing—the code generated by -Argument Clinic should be a "black box" to you, where CPython -calls in at the top, and your code gets called at the bottom, -with ``PyObject *args`` (and maybe ``PyObject *kwargs``) -magically converted into the C variables and types you need. - -In order for Argument Clinic to accomplish its primary goal, -it must be easy to use. Currently, working with CPython's -argument parsing library is a chore, requiring maintaining -redundant information in a surprising number of places. -When you use Argument Clinic, you don't have to repeat yourself. - -Obviously, no one would want to use Argument Clinic unless -it's solving their problem—and without creating new problems of -its own. -So it's paramount that Argument Clinic generate correct code. -It'd be nice if the code was faster, too, but at the very least -it should not introduce a major speed regression. (Eventually Argument -Clinic *should* make a major speedup possible—we could -rewrite its code generator to produce tailor-made argument -parsing code, rather than calling the general-purpose CPython -argument parsing library. That would make for the fastest -argument parsing possible!) - -Additionally, Argument Clinic must be flexible enough to -work with any approach to argument parsing. Python has -some functions with some very strange parsing behaviors; -Argument Clinic's goal is to support all of them. - -Finally, the original motivation for Argument Clinic was -to provide introspection "signatures" for CPython builtins. -It used to be, the introspection query functions would throw -an exception if you passed in a builtin. With Argument -Clinic, that's a thing of the past! - -One idea you should keep in mind, as you work with -Argument Clinic: the more information you give it, the -better job it'll be able to do. -Argument Clinic is admittedly relatively simple right -now. But as it evolves it will get more sophisticated, -and it should be able to do many interesting and smart -things with all the information you give it. - - Basic concepts -------------- @@ -1393,35 +1343,37 @@ state. Example from the ``setattro`` slot method in See also :pep:`573`. +.. _clinic-howto-custom-converter: + How to write a custom converter ------------------------------- -As we hinted at in the previous section... you can write your own converters! -A converter is simply a Python class that inherits from :py:class:`!CConverter`. -The main purpose of a custom converter is if you have a parameter using -the ``O&`` format unit—parsing this parameter means calling +A converter is a Python class that inherits from :py:class:`!CConverter`. +The main purpose of a custom converter, is for parameters parsed with +the ``O&`` format unit --- parsing such a parameter means calling a :c:func:`PyArg_ParseTuple` "converter function". -Your converter class should be named ``*something*_converter``. -If the name follows this convention, then your converter class -will be automatically registered with Argument Clinic; its name -will be the name of your class with the ``_converter`` suffix -stripped off. (This is accomplished with a metaclass.) +Your converter class should be named :samp:`{ConverterName}_converter`. +By following this convention, your converter class will be automatically +registered with Argument Clinic, with its *converter name* being the name of +your converter class with the ``_converter`` suffix stripped off. -You shouldn't subclass :py:meth:`!CConverter.__init__`. Instead, you should -write a :py:meth:`!converter_init` function. :py:meth:`!converter_init` -always accepts a *self* parameter; after that, all additional -parameters *must* be keyword-only. Any arguments passed in to -the converter in Argument Clinic will be passed along to your -:py:meth:`!converter_init`. - -There are some additional members of :py:class:`!CConverter` you may wish -to specify in your subclass. Here's the current list: +Instead of subclassing :py:meth:`!CConverter.__init__`, +write a :py:meth:`!converter_init` method. +Apart for the *self* parameter, all additional :py:meth:`!converter_init` +parameters **must** be keyword-only. +Any arguments passed to the converter in Argument Clinic +will be passed along to your :py:meth:`!converter_init` method. +See :py:class:`!CConverter` for a list of members you may wish to specify in +your subclass. .. module:: clinic .. class:: CConverter + The base class for all converters. + See :ref:`clinic-howto-custom-converter` for how to subclass this class. + .. attribute:: type The C type to use for this variable. @@ -1486,16 +1438,16 @@ Here's the simplest example of a custom converter, from :source:`Modules/zlibmod [python start generated code]*/ /*[python end generated code: output=da39a3ee5e6b4b0d input=35521e4e733823c7]*/ -This block adds a converter to Argument Clinic named ``ssize_t``. Parameters -declared as ``ssize_t`` will be declared as type :c:type:`Py_ssize_t`, and will -be parsed by the ``'O&'`` format unit, which will call the -``ssize_t_converter`` converter function. ``ssize_t`` variables -automatically support default values. +This block adds a converter named ``ssize_t`` to Argument Clinic. +Parameters declared as ``ssize_t`` will be declared with type :c:type:`Py_ssize_t`, +and will be parsed by the ``'O&'`` format unit, +which will call the :c:func:`!ssize_t_converter` converter C function. +``ssize_t`` variables automatically support default values. More sophisticated custom converters can insert custom C code to handle initialization and cleanup. You can see more examples of custom converters in the CPython -source tree; grep the C files for the string :py:class:`!CConverter`. +source tree; grep the C files for the string ``CConverter``. How to write a custom return converter diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 60854c3c034d73..2551fbe87b5c2a 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -1,5 +1,7 @@ .. highlight:: c +.. _isolating-extensions-howto: + *************************** Isolating Extension Modules *************************** diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index bc953645183fa0..fcf711efe0eb7d 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -72,8 +72,9 @@ Windows appends the usual ``.dll`` file suffix automatically. On Linux, it is required to specify the filename *including* the extension to load a library, so attribute access can not be used to load libraries. Either the -:meth:`LoadLibrary` method of the dll loaders should be used, or you should load -the library by creating an instance of CDLL by calling the constructor:: +:meth:`~LibraryLoader.LoadLibrary` method of the dll loaders should be used, +or you should load the library by creating an instance of CDLL by calling +the constructor:: >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX @@ -333,7 +334,7 @@ property:: 10 b'Hi\x00lo\x00\x00\x00\x00\x00' >>> -The :func:`create_string_buffer` function replaces the old :func:`c_buffer` +The :func:`create_string_buffer` function replaces the old :func:`!c_buffer` function (which is still available as an alias). To create a mutable memory block containing unicode characters of the C type :c:type:`wchar_t`, use the :func:`create_unicode_buffer` function. @@ -361,7 +362,7 @@ from within *IDLE* or *PythonWin*:: >>> printf(b"%f bottles of beer\n", 42.5) Traceback (most recent call last): File "", line 1, in - ArgumentError: argument 2: TypeError: Don't know how to convert parameter 2 + ctypes.ArgumentError: argument 2: TypeError: Don't know how to convert parameter 2 >>> As has been mentioned before, all Python types except integers, strings, and @@ -383,15 +384,15 @@ as calling functions with a fixed number of parameters. On some platforms, and i particular ARM64 for Apple Platforms, the calling convention for variadic functions is different than that for regular functions. -On those platforms it is required to specify the *argtypes* attribute for the -regular, non-variadic, function arguments: +On those platforms it is required to specify the :attr:`~_FuncPtr.argtypes` +attribute for the regular, non-variadic, function arguments: .. code-block:: python3 libc.printf.argtypes = [ctypes.c_char_p] Because specifying the attribute does not inhibit portability it is advised to always -specify ``argtypes`` for all variadic functions. +specify :attr:`~_FuncPtr.argtypes` for all variadic functions. .. _ctypes-calling-functions-with-own-custom-data-types: @@ -401,9 +402,9 @@ Calling functions with your own custom data types You can also customize :mod:`ctypes` argument conversion to allow instances of your own classes be used as function arguments. :mod:`ctypes` looks for an -:attr:`_as_parameter_` attribute and uses this as the function argument. The +:attr:`!_as_parameter_` attribute and uses this as the function argument. The attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an -object with an :attr:`_as_parameter_` attribute:: +object with an :attr:`!_as_parameter_` attribute:: >>> class Bottles: ... def __init__(self, number): @@ -415,7 +416,7 @@ object with an :attr:`_as_parameter_` attribute:: 19 >>> -If you don't want to store the instance's data in the :attr:`_as_parameter_` +If you don't want to store the instance's data in the :attr:`!_as_parameter_` instance variable, you could define a :class:`property` which makes the attribute available on request. @@ -426,9 +427,9 @@ Specifying the required argument types (function prototypes) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to specify the required argument types of functions exported from -DLLs by setting the :attr:`argtypes` attribute. +DLLs by setting the :attr:`~_FuncPtr.argtypes` attribute. -:attr:`argtypes` must be a sequence of C data types (the ``printf`` function is +:attr:`~_FuncPtr.argtypes` must be a sequence of C data types (the :func:`!printf` function is probably not a good example here, because it takes a variable number and different types of parameters depending on the format string, on the other hand this is quite handy to experiment with this feature):: @@ -445,21 +446,21 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> printf(b"%d %d %d", 1, 2, 3) Traceback (most recent call last): File "", line 1, in - ArgumentError: argument 2: TypeError: wrong type + ctypes.ArgumentError: argument 2: TypeError: 'int' object cannot be interpreted as ctypes.c_char_p >>> printf(b"%s %d %f\n", b"X", 2, 3) X 2 3.000000 13 >>> If you have defined your own classes which you pass to function calls, you have -to implement a :meth:`from_param` class method for them to be able to use them -in the :attr:`argtypes` sequence. The :meth:`from_param` class method receives +to implement a :meth:`~_CData.from_param` class method for them to be able to use them +in the :attr:`~_FuncPtr.argtypes` sequence. The :meth:`~_CData.from_param` class method receives the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the -object itself, its :attr:`_as_parameter_` attribute, or whatever you want to +object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to pass as the C function argument in this case. Again, the result should be an integer, string, bytes, a :mod:`ctypes` instance, or an object with an -:attr:`_as_parameter_` attribute. +:attr:`!_as_parameter_` attribute. .. _ctypes-return-types: @@ -479,13 +480,13 @@ By default functions are assumed to return the C :c:expr:`int` type. Other return types can be specified by setting the :attr:`restype` attribute of the function object. -The C prototype of ``time()`` is ``time_t time(time_t *)``. Because :c:type:`time_t` -might be of a different type than the default return type ``int``, you should -specify the ``restype``:: +The C prototype of :c:func:`time` is ``time_t time(time_t *)``. Because :c:type:`time_t` +might be of a different type than the default return type :c:expr:`int`, you should +specify the :attr:`!restype` attribute:: >>> libc.time.restype = c_time_t -The argument types can be specified using ``argtypes``:: +The argument types can be specified using :attr:`~_FuncPtr.argtypes`:: >>> libc.time.argtypes = (POINTER(c_time_t),) @@ -494,7 +495,7 @@ To call the function with a ``NULL`` pointer as first argument, use ``None``:: >>> print(libc.time(None)) # doctest: +SKIP 1150640792 -Here is a more advanced example, it uses the ``strchr`` function, which expects +Here is a more advanced example, it uses the :func:`strchr` function, which expects a string pointer and a char, and returns a pointer to a string:: >>> strchr = libc.strchr @@ -507,8 +508,8 @@ a string pointer and a char, and returns a pointer to a string:: None >>> -If you want to avoid the ``ord("x")`` calls above, you can set the -:attr:`argtypes` attribute, and the second argument will be converted from a +If you want to avoid the :func:`ord("x") ` calls above, you can set the +:attr:`~_FuncPtr.argtypes` attribute, and the second argument will be converted from a single character Python bytes object into a C char: .. doctest:: @@ -854,7 +855,7 @@ Type conversions ^^^^^^^^^^^^^^^^ Usually, ctypes does strict type checking. This means, if you have -``POINTER(c_int)`` in the :attr:`argtypes` list of a function or as the type of +``POINTER(c_int)`` in the :attr:`~_FuncPtr.argtypes` list of a function or as the type of a member field in a structure definition, only instances of exactly the same type are accepted. There are some exceptions to this rule, where ctypes accepts other objects. For example, you can pass compatible array instances instead of @@ -875,7 +876,7 @@ pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: >>> In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`argtypes`, an object of the pointed +(such as ``POINTER(c_int)``) in :attr:`_FuncPtr.argtypes`, an object of the pointed type (``c_int`` in this case) can be passed to the function. ctypes will apply the required :func:`byref` conversion in this case automatically. @@ -1438,7 +1439,7 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms ``dlopen`` or ``LoadLibrary`` +parameter, otherwise the underlying platforms :c:func:`!dlopen` or :c:func:`LoadLibrary` function is used to load the library into the process, and to get a handle to it. @@ -1523,8 +1524,8 @@ underscore to not clash with exported function names: Shared libraries can also be loaded by using one of the prefabricated objects, which are instances of the :class:`LibraryLoader` class, either by calling the -:meth:`LoadLibrary` method, or by retrieving the library as attribute of the -loader instance. +:meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as +attribute of the loader instance. .. class:: LibraryLoader(dlltype) @@ -1640,14 +1641,14 @@ They are instances of a private class: unspecified arguments as well. When a foreign function is called, each actual argument is passed to the - :meth:`from_param` class method of the items in the :attr:`argtypes` + :meth:`~_CData.from_param` class method of the items in the :attr:`argtypes` tuple, this method allows adapting the actual argument to an object that the foreign function accepts. For example, a :class:`c_char_p` item in the :attr:`argtypes` tuple will convert a string passed as argument into a bytes object using ctypes conversion rules. New: It is now possible to put items in argtypes which are not ctypes - types, but each item must have a :meth:`from_param` method which returns a + types, but each item must have a :meth:`~_CData.from_param` method which returns a value usable as argument (integer, string, ctypes instance). This allows defining adapters that can adapt custom objects as function parameters. @@ -1771,12 +1772,12 @@ different ways, depending on the type and number of the parameters in the call: COM methods use a special calling convention: They require a pointer to the COM interface as first argument, in addition to those parameters that - are specified in the :attr:`argtypes` tuple. + are specified in the :attr:`~_FuncPtr.argtypes` tuple. The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above. - *paramflags* must be a tuple of the same length as :attr:`argtypes`. + *paramflags* must be a tuple of the same length as :attr:`~_FuncPtr.argtypes`. Each item in this tuple contains further information about a parameter, it must be a tuple containing one, two, or three items. @@ -2158,8 +2159,8 @@ Data types This method adapts *obj* to a ctypes type. It is called with the actual object used in a foreign function call when the type is present in the - foreign function's :attr:`argtypes` tuple; it must return an object that - can be used as a function call parameter. + foreign function's :attr:`~_FuncPtr.argtypes` tuple; + it must return an object that can be used as a function call parameter. All ctypes data types have a default implementation of this classmethod that normally returns *obj* if that is an instance of the type. Some diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index accc652a90649d..2217c3a8035459 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -922,9 +922,10 @@ iterations of the loop. .. opcode:: UNPACK_SEQUENCE (count) Unpacks ``STACK[-1]`` into *count* individual values, which are put onto the stack - right-to-left:: + right-to-left. Require there to be exactly *count* values.:: - STACK.extend(STACK.pop()[:count:-1]) + assert(len(STACK[-1]) == count) + STACK.extend(STACK.pop()[:-count-1:-1]) .. opcode:: UNPACK_EX (counts) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 2688f43f9b4ffc..b271067ae639c5 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -14,8 +14,8 @@ are always available. They are listed here in alphabetical order. | | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | | | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` | | | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | -| | :func:`any` | | | | | | :func:`round` | -| | :func:`anext` | | **F** | | **M** | | | +| | :func:`anext` | | | | | | :func:`round` | +| | :func:`any` | | **F** | | **M** | | | | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 01dabe286969bb..22360b22fd924b 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -976,6 +976,11 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.13 The *follow_symlinks* parameter was added. + .. versionchanged:: 3.13 + Emits :exc:`FutureWarning` if the pattern ends with "``**``". In a + future Python release, patterns with this ending will match both files + and directories. Add a trailing slash to match only directories. + .. method:: Path.group() Return the name of the group owning the file. :exc:`KeyError` is raised diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index e2791bbc97b002..fad945ffc8210a 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -849,6 +849,31 @@ using ``[]``. concat(b"foo", b"bar") # OK, output has type 'bytes' concat("foo", b"bar") # Error, cannot mix str and bytes + Note that, despite its name, ``AnyStr`` has nothing to do with the + :class:`Any` type, nor does it mean "any string". In particular, ``AnyStr`` + and ``str | bytes`` are different from each other and have different use + cases:: + + # Invalid use of AnyStr: + # The type variable is used only once in the function signature, + # so cannot be "solved" by the type checker + def greet_bad(cond: bool) -> AnyStr: + return "hi there!" if cond else b"greetings!" + + # The better way of annotating this function: + def greet_proper(cond: bool) -> str | bytes: + return "hi there!" if cond else b"greetings!" + + .. deprecated-removed:: 3.13 3.18 + Deprecated in favor of the new :ref:`type parameter syntax `. + Use ``class A[T: (str, bytes)]: ...`` instead of importing ``AnyStr``. See + :pep:`695` for more details. + + In Python 3.16, ``AnyStr`` will be removed from ``typing.__all__``, and + deprecation warnings will be emitted at runtime when it is accessed or + imported from ``typing``. ``AnyStr`` will be removed from ``typing`` + in Python 3.18. + .. data:: LiteralString Special type that includes only literal strings. @@ -938,13 +963,17 @@ using ``[]``. For example:: - from typing import Self + from typing import Self, reveal_type class Foo: def return_self(self) -> Self: ... return self + class SubclassOfFoo(Foo): pass + + reveal_type(Foo().return_self()) # Revealed type is "Foo" + reveal_type(SubclassOfFoo().return_self()) # Revealed type is "SubclassOfFoo" This annotation is semantically equivalent to the following, albeit in a more succinct fashion:: @@ -958,15 +987,11 @@ using ``[]``. ... return self - In general if something currently follows the pattern of:: - - class Foo: - def return_self(self) -> "Foo": - ... - return self - - You should use :data:`Self` as calls to ``SubclassOfFoo.return_self`` would have - ``Foo`` as the return type and not ``SubclassOfFoo``. + In general, if something returns ``self``, as in the above examples, you + should use ``Self`` as the return annotation. If ``Foo.return_self`` was + annotated as returning ``"Foo"``, then the type checker would infer the + object returned from ``SubclassOfFoo.return_self`` as being of type ``Foo`` + rather than ``SubclassOfFoo``. Other common use cases include: @@ -974,6 +999,17 @@ using ``[]``. of the ``cls`` parameter. - Annotating an :meth:`~object.__enter__` method which returns self. + You should not use ``Self`` as the return annotation if the method is not + guaranteed to return an instance of a subclass when the class is + subclassed:: + + class Eggs: + # Self would be an incorrect return annotation here, + # as the object returned is always an instance of Eggs, + # even in subclasses + def returns_eggs(self) -> "Eggs": + return Eggs() + See :pep:`673` for more details. .. versionadded:: 3.11 @@ -3685,3 +3721,7 @@ convenience. This is subject to change, and not all deprecations are listed. - 3.13 - 3.15 - :gh:`106309` + * - :data:`typing.AnyStr` + - 3.13 + - 3.18 + - :gh:`105578` diff --git a/Doc/license.rst b/Doc/license.rst index 1dadb6264a0059..be8efa70611378 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -659,134 +659,186 @@ The modules :mod:`hashlib`, :mod:`posix` and :mod:`ssl` use the OpenSSL library for added performance if made available by the operating system. Additionally, the Windows and macOS installers for Python may include a copy of the OpenSSL libraries, so we include a copy -of the OpenSSL license here:: - - - LICENSE ISSUES - ============== - - The OpenSSL toolkit stays under a dual license, i.e. both the conditions of - the OpenSSL License and the original SSLeay license apply to the toolkit. - See below for the actual license texts. Actually both licenses are BSD-style - Open Source licenses. In case of any license issues related to OpenSSL - please contact openssl-core@openssl.org. - - OpenSSL License - --------------- - - /* ==================================================================== - * Copyright (c) 1998-2008 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * openssl-core@openssl.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.openssl.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - * - */ - - Original SSLeay License - ----------------------- - - /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) - * All rights reserved. - * - * This package is an SSL implementation written - * by Eric Young (eay@cryptsoft.com). - * The implementation was written so as to conform with Netscapes SSL. - * - * This library is free for commercial and non-commercial use as long as - * the following conditions are aheared to. The following conditions - * apply to all code found in this distribution, be it the RC4, RSA, - * lhash, DES, etc., code; not just the SSL code. The SSL documentation - * included with this distribution is covered by the same copyright terms - * except that the holder is Tim Hudson (tjh@cryptsoft.com). - * - * Copyright remains Eric Young's, and as such any Copyright notices in - * the code are not to be removed. - * If this package is used in a product, Eric Young should be given attribution - * as the author of the parts of the library used. - * This can be in the form of a textual message at program startup or - * in documentation (online or textual) provided with the package. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * "This product includes cryptographic software written by - * Eric Young (eay@cryptsoft.com)" - * The word 'cryptographic' can be left out if the rouines from the library - * being used are not cryptographic related :-). - * 4. If you include any Windows specific code (or a derivative thereof) from - * the apps directory (application code) you must include an acknowledgement: - * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" - * - * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * The licence and distribution terms for any publically available version or - * derivative of this code cannot be changed. i.e. this code cannot simply be - * copied and put under another distribution licence - * [including the GNU Public Licence.] - */ +of the OpenSSL license here. For the OpenSSL 3.0 release, +and later releases derived from that, the Apache License v2 applies:: + + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS expat diff --git a/Doc/requirements.txt b/Doc/requirements.txt index bde509febf5bde..e9b6e56b122895 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -7,7 +7,7 @@ # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. # PR #104777: Sphinx 6.2 no longer uses imghdr, removed in Python 3.13. -sphinx==6.2.0 +sphinx==6.2.1 blurb @@ -15,6 +15,6 @@ sphinxext-opengraph==0.7.5 # The theme used by the documentation is stored separately, so we need # to install that as well. -python-docs-theme>=2022.1 +python-docs-theme>=2023.7 -c constraints.txt diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8d99b0bfa4f381..765e6383ac6f5e 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -98,14 +98,13 @@ class ImplementationDetail(Directive): final_argument_whitespace = True # This text is copied to templates/dummy.html - label_text = 'CPython implementation detail:' + label_text = sphinx_gettext('CPython implementation detail:') def run(self): self.assert_has_content() pnode = nodes.compound(classes=['impl-detail']) - label = sphinx_gettext(self.label_text) content = self.content - add_text = nodes.strong(label, label) + add_text = nodes.strong(self.label_text, self.label_text) self.state.nested_parse(content, self.content_offset, pnode) content = nodes.inline(pnode[0].rawsource, translatable=True) content.source = pnode[0].source @@ -234,9 +233,9 @@ class AuditEvent(Directive): final_argument_whitespace = True _label = [ - "Raises an :ref:`auditing event ` {name} with no arguments.", - "Raises an :ref:`auditing event ` {name} with argument {args}.", - "Raises an :ref:`auditing event ` {name} with arguments {args}.", + sphinx_gettext("Raises an :ref:`auditing event ` {name} with no arguments."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with argument {args}."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with arguments {args}."), ] @property @@ -252,7 +251,7 @@ def run(self): else: args = [] - label = sphinx_gettext(self._label[min(2, len(args))]) + label = self._label[min(2, len(args))] text = label.format(name="``{}``".format(name), args=", ".join("``{}``".format(a) for a in args if a)) @@ -414,8 +413,8 @@ class DeprecatedRemoved(Directive): final_argument_whitespace = True option_spec = {} - _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}' - _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}' + _deprecated_label = sphinx_gettext('Deprecated since version {deprecated}, will be removed in version {removed}') + _removed_label = sphinx_gettext('Deprecated since version {deprecated}, removed in version {removed}') def run(self): node = addnodes.versionmodified() @@ -431,7 +430,6 @@ def run(self): else: label = self._removed_label - label = sphinx_gettext(label) text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 9832feba141675..80103158ea01e6 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,7 +26,9 @@ {% endblock %} {% block extrahead %} - + {% if builder == "html" %} + + {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 99500e1f321468..6553013552d003 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -69,6 +69,10 @@ New grammar features: * :pep:`701`: Syntactic formalization of f-strings +Interpreter improvements: + +* :ref:`whatsnew312-pep684` + New typing features: * :pep:`688`: Making the buffer protocol accessible in Python @@ -276,6 +280,36 @@ The new :class:`inspect.BufferFlags` enum represents the flags that can be used to customize buffer creation. (Contributed by Jelle Zijlstra in :gh:`102500`.) +.. _whatsnew312-pep684: + +PEP 684: A Per-Interpreter GIL +------------------------------ + +Sub-interpreters may now be created with a unique GIL per interpreter. +This allows Python programs to take full advantage of multiple CPU +cores. + +Use the new :c:func:`Py_NewInterpreterFromConfig` function to +create an interpreter with its own GIL:: + + PyInterpreterConfig config = { + .check_multi_interp_extensions = 1, + .gil = PyInterpreterConfig_OWN_GIL, + }; + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + if (PyStatus_Exception(status)) { + return -1; + } + /* The new interpeter is now active in the current thread. */ + +For further examples how to use the C-API for sub-interpreters with a +per-interpreter GIL, see :source:`Modules/_xxsubinterpretersmodule.c`. + +A Python API is anticipated for 3.13. (See :pep:`554`.) + +(Contributed by Eric Snow in :gh:`104210`, etc.) + New Features Related to Type Hints ================================== @@ -767,6 +801,11 @@ sys exception instance, rather than to a ``(typ, exc, tb)`` tuple. (Contributed by Irit Katriel in :gh:`103176`.) +* :func:`sys.setrecursionlimit` and :func:`sys.getrecursionlimit`. + The recursion limit now applies only to Python code. Builtin functions do + not use the recursion limit, but are protected by a different mechanism + that prevents recursion from causing a virtual machine crash. + tempfile -------- @@ -1234,6 +1273,10 @@ Removed (Contributed by Pradyun Gedam in :gh:`95299`.) +* :mod:`enum`: Remove ``EnumMeta.__getattr__``, which is no longer needed for + enum attribute access. + (Contributed by Ethan Furman in :gh:`95083`.) + * :mod:`ftplib`: Remove the ``FTP_TLS.ssl_version`` class attribute: use the *context* parameter instead. (Contributed by Victor Stinner in :gh:`94172`.) @@ -1738,6 +1781,12 @@ New Features (Contributed by Eddie Elizondo in :gh:`84436`.) +* :pep:`684`: Added the new :c:func:`Py_NewInterpreterFromConfig` + function and :c:type:`PyInterpreterConfig`, which may be used + to create sub-interpreters with their own GILs. + (See :ref:`whatsnew312-pep684` for more info.) + (Contributed by Eric Snow in :gh:`104110`.) + * In the limited C API version 3.12, :c:func:`Py_INCREF` and :c:func:`Py_DECREF` functions are now implemented as opaque function calls to hide implementation details. @@ -1876,6 +1925,13 @@ Porting to Python 3.12 * :c:func:`PyUnstable_Long_IsCompact` * :c:func:`PyUnstable_Long_CompactValue` +* Custom allocators, set via :c:func:`PyMem_SetAllocator`, are now + required to be thread-safe, regardless of memory domain. Allocators + that don't have their own state, including "hooks", are not affected. + If your custom allocator is not already thread-safe and you need + guidance then please create a new GitHub issue + and CC ``@ericsnowcurrently``. + Deprecated ---------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index e20832a3a4c309..63cdee6cf1a4f3 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -117,6 +117,18 @@ and only logged in :ref:`Python Development Mode ` or on :ref:`Python built on debug mode `. (Contributed by Victor Stinner in :gh:`62948`.) +opcode +------ + +* Move ``opcode.ENABLE_SPECIALIZATION`` to ``_opcode.ENABLE_SPECIALIZATION``. + This field was added in 3.12, it was never documented and is not intended for + external usage. (Contributed by Irit Katriel in :gh:`105481`.) + +* Removed ``opcode.is_pseudo``, ``opcode.MIN_PSEUDO_OPCODE`` and + ``opcode.MAX_PSEUDO_OPCODE``, which were added in 3.12, were never + documented or exposed through ``dis``, and were not intended to be + used externally. + pathlib ------- @@ -196,6 +208,12 @@ Deprecated has yet to be supported by any major type checkers. (Contributed by Alex Waygood in :gh:`106309`.) + * :data:`typing.AnyStr` is deprecated. In Python 3.16, it will be removed from + ``typing.__all__``, and a :exc:`DeprecationWarning` will be emitted when it + is imported or accessed. It will be removed entirely in Python 3.18. Use + the new :ref:`type parameter syntax ` instead. + (Contributed by Michael The in :gh:`107116`.) + * :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. They will be removed in Python 3.15. diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 2260501bfd608e..da34ec1882a539 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -12,7 +12,7 @@ typedef struct { } _PyVMData; typedef struct _PyExecutorObject { - PyObject_HEAD + PyObject_VAR_HEAD /* WARNING: execute consumes a reference to self. This is necessary to allow executors to tail call into each other. */ struct _PyInterpreterFrame *(*execute)(struct _PyExecutorObject *self, struct _PyInterpreterFrame *frame, PyObject **stack_pointer); _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 30de4ee4b6c58c..56e473cc5e42d5 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -222,7 +222,8 @@ struct _ts { # ifdef __wasi__ # define C_RECURSION_LIMIT 500 # else -# define C_RECURSION_LIMIT 800 + // This value is duplicated in Lib/test/support/__init__.py +# define C_RECURSION_LIMIT 1500 # endif #endif diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index bcdf8645c8557c..00099376635e9b 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -229,6 +229,8 @@ extern void _PyLineTable_InitAddressRange( extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); +#define ENABLE_SPECIALIZATION 1 + /* Specialization functions */ extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, @@ -272,6 +274,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0) +#define GC_STAT_ADD(gen, name, n) do { if (_py_stats) _py_stats->gc_stats[(gen)].name += (n); } while (0) // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -285,6 +288,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OBJECT_STAT_INC_COND(name, cond) ((void)0) #define EVAL_CALL_STAT_INC(name) ((void)0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0) +#define GC_STAT_ADD(gen, name, n) ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 3f8d8adc83b20a..3dc00ec7e04c6f 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -64,19 +64,19 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); # error "_PY_NSMALLPOSINTS must be greater than or equal to 257" #endif -// Return a borrowed reference to the zero singleton. +// Return a reference to the immortal zero singleton. // The function cannot return NULL. static inline PyObject* _PyLong_GetZero(void) { return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS]; } -// Return a borrowed reference to the one singleton. +// Return a reference to the immortal one singleton. // The function cannot return NULL. static inline PyObject* _PyLong_GetOne(void) { return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+1]; } static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i) { - return Py_NewRef((PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+i]); + return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+i]; } extern PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b730aba9912830..76b3cd69cbf5a7 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -292,8 +292,8 @@ extern void _PyDebug_PrintTotalRefs(void); #ifdef Py_TRACE_REFS extern void _Py_AddToAllObjects(PyObject *op, int force); -extern void _Py_PrintReferences(FILE *); -extern void _Py_PrintReferenceAddresses(FILE *); +extern void _Py_PrintReferences(PyInterpreterState *, FILE *); +extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *); #endif diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index 94005d77881432..65feb5af969f8b 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -11,17 +11,22 @@ extern "C" { struct _py_object_runtime_state { #ifdef Py_REF_DEBUG Py_ssize_t interpreter_leaks; -#else - int _not_used; #endif + int _not_used; }; struct _py_object_state { #ifdef Py_REF_DEBUG Py_ssize_t reftotal; -#else - int _not_used; #endif +#ifdef Py_TRACE_REFS + /* Head of circular doubly-linked list of all objects. These are linked + * together via the _ob_prev and _ob_next members of a PyObject, which + * exist only in a Py_TRACE_REFS build. + */ + PyObject refchain; +#endif + int _not_used; }; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d525913f8a7aba..1cab6c984f3ace 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -35,27 +35,26 @@ #define _BINARY_OP_ADD_UNICODE 311 #define _LOAD_LOCALS 312 #define _LOAD_FROM_DICT_OR_GLOBALS 313 -#define _SKIP_CACHE 314 -#define _GUARD_GLOBALS_VERSION 315 -#define _GUARD_BUILTINS_VERSION 316 -#define _LOAD_GLOBAL_MODULE 317 -#define _LOAD_GLOBAL_BUILTINS 318 -#define _GUARD_TYPE_VERSION 319 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 320 -#define _LOAD_ATTR_INSTANCE_VALUE 321 -#define IS_NONE 322 -#define _ITER_CHECK_LIST 323 -#define _IS_ITER_EXHAUSTED_LIST 324 -#define _ITER_NEXT_LIST 325 -#define _ITER_CHECK_TUPLE 326 -#define _IS_ITER_EXHAUSTED_TUPLE 327 -#define _ITER_NEXT_TUPLE 328 -#define _ITER_CHECK_RANGE 329 -#define _IS_ITER_EXHAUSTED_RANGE 330 -#define _ITER_NEXT_RANGE 331 -#define _POP_JUMP_IF_FALSE 332 -#define _POP_JUMP_IF_TRUE 333 -#define JUMP_TO_TOP 334 +#define _GUARD_GLOBALS_VERSION 314 +#define _GUARD_BUILTINS_VERSION 315 +#define _LOAD_GLOBAL_MODULE 316 +#define _LOAD_GLOBAL_BUILTINS 317 +#define _GUARD_TYPE_VERSION 318 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 +#define _LOAD_ATTR_INSTANCE_VALUE 320 +#define IS_NONE 321 +#define _ITER_CHECK_LIST 322 +#define _IS_ITER_EXHAUSTED_LIST 323 +#define _ITER_NEXT_LIST 324 +#define _ITER_CHECK_TUPLE 325 +#define _IS_ITER_EXHAUSTED_TUPLE 326 +#define _ITER_NEXT_TUPLE 327 +#define _ITER_CHECK_RANGE 328 +#define _IS_ITER_EXHAUSTED_RANGE 329 +#define _ITER_NEXT_RANGE 330 +#define _POP_JUMP_IF_FALSE 331 +#define _POP_JUMP_IF_TRUE 332 +#define JUMP_TO_TOP 333 #ifndef NEED_OPCODE_METADATA extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); @@ -680,9 +679,9 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case LOAD_GLOBAL: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_GLOBAL_MODULE: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case LOAD_GLOBAL_BUILTIN: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case DELETE_FAST: return 0; case MAKE_CELL: @@ -740,7 +739,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case LOAD_METHOD: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_INSTANCE_VALUE: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case LOAD_ATTR_MODULE: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_WITH_HINT: @@ -945,7 +944,18 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { } #endif -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000, INSTR_FMT_IBC00000000, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC0, INSTR_FMT_IXC00, INSTR_FMT_IXC000 }; +enum InstructionFormat { + INSTR_FMT_IB, + INSTR_FMT_IBC, + INSTR_FMT_IBC00, + INSTR_FMT_IBC000, + INSTR_FMT_IBC00000000, + INSTR_FMT_IX, + INSTR_FMT_IXC, + INSTR_FMT_IXC0, + INSTR_FMT_IXC00, + INSTR_FMT_IXC000, +}; #define IS_VALID_OPCODE(OP) \ (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \ @@ -1279,8 +1289,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_NAME] = { .nuops = 2, .uops = { { _LOAD_LOCALS, 0, 0 }, { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, [LOAD_GLOBAL] = { .nuops = 1, .uops = { { LOAD_GLOBAL, 0, 0 } } }, - [LOAD_GLOBAL_MODULE] = { .nuops = 4, .uops = { { _SKIP_CACHE, 0, 0 }, { _GUARD_GLOBALS_VERSION, 1, 1 }, { _SKIP_CACHE, 0, 0 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, - [LOAD_GLOBAL_BUILTIN] = { .nuops = 4, .uops = { { _SKIP_CACHE, 0, 0 }, { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, + [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, + [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } }, [DELETE_DEREF] = { .nuops = 1, .uops = { { DELETE_DEREF, 0, 0 } } }, [LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_DEREF, 0, 0 } } }, @@ -1302,7 +1312,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { LOAD_ATTR, 0, 0 } } }, - [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 4, .uops = { { _SKIP_CACHE, 0, 0 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, + [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { COMPARE_OP, 0, 0 } } }, [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } }, [COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } }, @@ -1356,7 +1366,6 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_LOAD_LOCALS] = "_LOAD_LOCALS", [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", - [_SKIP_CACHE] = "_SKIP_CACHE", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index dcfceef4f175a2..e89d368be07aa7 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -157,6 +157,7 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 10, }, \ }, \ }, \ + .object_state = _py_object_state_INIT(INTERP), \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ .func_state = { \ @@ -186,6 +187,16 @@ extern PyTypeObject _PyExc_MemoryError; .context_ver = 1, \ } +#ifdef Py_TRACE_REFS +# define _py_object_state_INIT(INTERP) \ + { \ + .refchain = {&INTERP.object_state.refchain, &INTERP.object_state.refchain}, \ + } +#else +# define _py_object_state_INIT(INTERP) \ + { 0 } +#endif + // global objects diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index edb141cc79f752..57a5970353b360 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -18,7 +18,7 @@ typedef struct { typedef struct { _PyExecutorObject base; - _PyUOpInstruction trace[_Py_UOP_MAX_TRACE_LENGTH]; // TODO: variable length + _PyUOpInstruction trace[1]; } _PyUOpExecutorObject; _PyInterpreterFrame *_PyUopExecute( diff --git a/Include/opcode.h b/Include/opcode.h index 697520937d9055..ede1518b6fd25c 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -146,7 +146,6 @@ extern "C" { #define INSTRUMENTED_END_SEND 252 #define INSTRUMENTED_INSTRUCTION 253 #define INSTRUMENTED_LINE 254 -#define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 #define SETUP_WITH 258 @@ -159,7 +158,6 @@ extern "C" { #define LOAD_ZERO_SUPER_ATTR 265 #define STORE_FAST_MAYBE_NULL 266 #define LOAD_CLOSURE 267 -#define MAX_PSEUDO_OPCODE 267 #define TO_BOOL_ALWAYS_TRUE 7 #define TO_BOOL_BOOL 8 #define TO_BOOL_INT 10 @@ -256,8 +254,6 @@ extern "C" { #define NB_INPLACE_TRUE_DIVIDE 24 #define NB_INPLACE_XOR 25 -/* Defined in Lib/opcode.py */ -#define ENABLE_SPECIALIZATION 1 #ifdef __cplusplus } diff --git a/Include/pystats.h b/Include/pystats.h index 54c9b8d8b3538f..e24aef5fe8072b 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -74,12 +74,21 @@ typedef struct _object_stats { uint64_t optimization_traces_created; uint64_t optimization_traces_executed; uint64_t optimization_uops_executed; + /* Temporary value used during GC */ + uint64_t object_visits; } ObjectStats; +typedef struct _gc_stats { + uint64_t collections; + uint64_t object_visits; + uint64_t objects_collected; +} GCStats; + typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; ObjectStats object_stats; + GCStats *gc_stats; } PyStats; diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index bf15f517e50dce..b7ad365709b19e 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -5,6 +5,7 @@ import collections import socket import sys +import warnings import weakref if hasattr(socket, 'AF_UNIX'): @@ -392,6 +393,11 @@ async def start_tls(self, sslcontext, *, self._transport = new_transport protocol._replace_writer(self) + def __del__(self, warnings=warnings): + if not self._transport.is_closing(): + self.close() + warnings.warn(f"unclosed {self!r}", ResourceWarning) + class StreamReader: diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index af1d5c4800cce8..f5aba434fd4253 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1923,8 +1923,40 @@ def __init__(self, address): self._ip = self._ip_int_from_string(addr_str) + def _explode_shorthand_ip_string(self): + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + long_form = super()._explode_shorthand_ip_string() + else: + prefix_len = 30 + raw_exploded_str = super()._explode_shorthand_ip_string() + long_form = "%s%s" % (raw_exploded_str[:prefix_len], str(ipv4_mapped)) + return long_form + + def _ipv4_mapped_ipv6_to_str(self): + """Return convenient text representation of IPv4-mapped IPv6 address + + See RFC 4291 2.5.5.2, 2.2 p.3 for details. + + Returns: + A string, 'x:x:x:x:x:x:d.d.d.d', where the 'x's are the hexadecimal values of + the six high-order 16-bit pieces of the address, and the 'd's are + the decimal values of the four low-order 8-bit pieces of the + address (standard IPv4 representation) as defined in RFC 4291 2.2 p.3. + + """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + raise AddressValueError("Can not apply to non-IPv4-mapped IPv6 address %s" % str(self)) + high_order_bits = self._ip >> 32 + return "%s:%s" % (self._string_from_ip_int(high_order_bits), str(ipv4_mapped)) + def __str__(self): - ip_str = super().__str__() + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + ip_str = super().__str__() + else: + ip_str = self._ipv4_mapped_ipv6_to_str() return ip_str + '%' + self._scope_id if self._scope_id else ip_str def __hash__(self): diff --git a/Lib/opcode.py b/Lib/opcode.py index bed922399821a6..5a9f8ddd0738db 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -19,26 +19,11 @@ cmp_op = ('<', '<=', '==', '!=', '>', '>=') - -ENABLE_SPECIALIZATION = True - -def is_pseudo(op): - return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE - opmap = {} -# pseudo opcodes (used in the compiler) mapped to the values -# they can become in the actual code. -_pseudo_ops = {} - def def_op(name, op): opmap[name] = op -def pseudo_op(name, op, real_ops): - def_op(name, op) - _pseudo_ops[name] = real_ops - - # Instruction opcodes for compiled code # Blank lines correspond to available opcodes @@ -215,29 +200,27 @@ def pseudo_op(name, op, real_ops): # 255 is reserved -MIN_PSEUDO_OPCODE = 256 - -pseudo_op('SETUP_FINALLY', 256, ['NOP']) -pseudo_op('SETUP_CLEANUP', 257, ['NOP']) -pseudo_op('SETUP_WITH', 258, ['NOP']) -pseudo_op('POP_BLOCK', 259, ['NOP']) +# Pseudo ops are above 255: -pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) -pseudo_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT']) +def_op('SETUP_FINALLY', 256) +def_op('SETUP_CLEANUP', 257) +def_op('SETUP_WITH', 258) +def_op('POP_BLOCK', 259) -pseudo_op('LOAD_METHOD', 262, ['LOAD_ATTR']) -pseudo_op('LOAD_SUPER_METHOD', 263, ['LOAD_SUPER_ATTR']) -pseudo_op('LOAD_ZERO_SUPER_METHOD', 264, ['LOAD_SUPER_ATTR']) -pseudo_op('LOAD_ZERO_SUPER_ATTR', 265, ['LOAD_SUPER_ATTR']) +def_op('JUMP', 260) +def_op('JUMP_NO_INTERRUPT', 261) -pseudo_op('STORE_FAST_MAYBE_NULL', 266, ['STORE_FAST']) -pseudo_op('LOAD_CLOSURE', 267, ['LOAD_FAST']) +def_op('LOAD_METHOD', 262) +def_op('LOAD_SUPER_METHOD', 263) +def_op('LOAD_ZERO_SUPER_METHOD', 264) +def_op('LOAD_ZERO_SUPER_ATTR', 265) -MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 +def_op('STORE_FAST_MAYBE_NULL', 266) +def_op('LOAD_CLOSURE', 267) -del def_op, pseudo_op +del def_op -opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] +opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] for op, i in opmap.items(): opname[i] = op diff --git a/Lib/pathlib.py b/Lib/pathlib.py index c83cf3d2ef696d..758f70f5ba7077 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1069,6 +1069,11 @@ def _glob(self, pattern, case_sensitive, follow_symlinks): pattern_parts.append('') if pattern_parts[-1] == '**': # GH-70303: '**' only matches directories. Add trailing slash. + warnings.warn( + "Pattern ending '**' will match files and directories in a " + "future Python release. Add a trailing slash to match only " + "directories and remove this warning.", + FutureWarning, 3) pattern_parts.append('') if case_sensitive is None: diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index a660ccf8876e2d..d2ad1a0482c304 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5004,3 +5004,463 @@ PyDoc_STRVAR(new_dest__doc__, "\n" "Only this docstring should be outputted to test1."); /*[clinic end generated code: output=9cac703f51d90e84 input=090db8df4945576d]*/ + + +/*[clinic input] +mangled_c_keyword_identifier + i as int: int +The 'int' param should be mangled as 'int_value' +[clinic start generated code]*/ + +PyDoc_STRVAR(mangled_c_keyword_identifier__doc__, +"mangled_c_keyword_identifier($module, /, i)\n" +"--\n" +"\n" +"The \'int\' param should be mangled as \'int_value\'"); + +#define MANGLED_C_KEYWORD_IDENTIFIER_METHODDEF \ + {"mangled_c_keyword_identifier", _PyCFunction_CAST(mangled_c_keyword_identifier), METH_FASTCALL|METH_KEYWORDS, mangled_c_keyword_identifier__doc__}, + +static PyObject * +mangled_c_keyword_identifier_impl(PyObject *module, int int_value); + +static PyObject * +mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(i), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"i", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "mangled_c_keyword_identifier", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int int_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + int_value = _PyLong_AsInt(args[0]); + if (int_value == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = mangled_c_keyword_identifier_impl(module, int_value); + +exit: + return return_value; +} + +static PyObject * +mangled_c_keyword_identifier_impl(PyObject *module, int int_value) +/*[clinic end generated code: output=c049d7d79be26cda input=060876448ab567a2]*/ + + +/*[clinic input] +bool_return -> bool +[clinic start generated code]*/ + +PyDoc_STRVAR(bool_return__doc__, +"bool_return($module, /)\n" +"--\n" +"\n"); + +#define BOOL_RETURN_METHODDEF \ + {"bool_return", (PyCFunction)bool_return, METH_NOARGS, bool_return__doc__}, + +static int +bool_return_impl(PyObject *module); + +static PyObject * +bool_return(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = bool_return_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +static int +bool_return_impl(PyObject *module) +/*[clinic end generated code: output=3a65f07830e48e98 input=93ba95d39ee98f39]*/ + + +/*[clinic input] +double_return -> double +[clinic start generated code]*/ + +PyDoc_STRVAR(double_return__doc__, +"double_return($module, /)\n" +"--\n" +"\n"); + +#define DOUBLE_RETURN_METHODDEF \ + {"double_return", (PyCFunction)double_return, METH_NOARGS, double_return__doc__}, + +static double +double_return_impl(PyObject *module); + +static PyObject * +double_return(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + double _return_value; + + _return_value = double_return_impl(module); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + +static double +double_return_impl(PyObject *module) +/*[clinic end generated code: output=076dc72595d3f66d input=da11b6255e4cbfd7]*/ + + +/*[clinic input] +Test.__init__ + a: object + [ + b: object + ] + / +Should generate two PyArg_ParseTuple calls. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test(a, [b])\n" +"Should generate two PyArg_ParseTuple calls."); + +static int +Test___init___impl(TestObj *self, PyObject *a, int group_right_1, + PyObject *b); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + PyObject *a; + int group_right_1 = 0; + PyObject *b = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + switch (PyTuple_GET_SIZE(args)) { + case 1: + if (!PyArg_ParseTuple(args, "O:__init__", &a)) { + goto exit; + } + break; + case 2: + if (!PyArg_ParseTuple(args, "OO:__init__", &a, &b)) { + goto exit; + } + group_right_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "Test.__init__ requires 1 to 2 arguments"); + goto exit; + } + return_value = Test___init___impl((TestObj *)self, a, group_right_1, b); + +exit: + return return_value; +} + +static int +Test___init___impl(TestObj *self, PyObject *a, int group_right_1, + PyObject *b) +/*[clinic end generated code: output=2bbb8ea60e8f57a6 input=10f5d0f1e8e466ef]*/ + + +/*[clinic input] +Test._pyarg_parsestackandkeywords + cls: defining_class + key: str(accept={str, robuffer}, zeroes=True) + / +Check that _PyArg_ParseStackAndKeywords() is generated. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test__pyarg_parsestackandkeywords__doc__, +"_pyarg_parsestackandkeywords($self, key, /)\n" +"--\n" +"\n" +"Check that _PyArg_ParseStackAndKeywords() is generated."); + +#define TEST__PYARG_PARSESTACKANDKEYWORDS_METHODDEF \ + {"_pyarg_parsestackandkeywords", _PyCFunction_CAST(Test__pyarg_parsestackandkeywords), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, Test__pyarg_parsestackandkeywords__doc__}, + +static PyObject * +Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, + const char *key, + Py_ssize_t key_length); + +static PyObject * +Test__pyarg_parsestackandkeywords(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "s#:_pyarg_parsestackandkeywords", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + const char *key; + Py_ssize_t key_length; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &key, &key_length)) { + goto exit; + } + return_value = Test__pyarg_parsestackandkeywords_impl(self, cls, key, key_length); + +exit: + return return_value; +} + +static PyObject * +Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, + const char *key, + Py_ssize_t key_length) +/*[clinic end generated code: output=4fda8a7f2547137c input=fc72ef4b4cfafabc]*/ + + +/*[clinic input] +Test.__init__ -> long +Test overriding the __init__ return converter +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test()\n" +"--\n" +"\n" +"Test overriding the __init__ return converter"); + +static long +Test___init___impl(TestObj *self); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + long _return_value; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoPositional("Test", args)) { + goto exit; + } + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + _return_value = Test___init___impl((TestObj *)self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong(_return_value); + +exit: + return return_value; +} + +static long +Test___init___impl(TestObj *self) +/*[clinic end generated code: output=daf6ee12c4e443fb input=311af0dc7f17e8e9]*/ + + +/*[clinic input] +fn_with_default_binop_expr + arg: object(c_default='CONST_A + CONST_B') = a+b +[clinic start generated code]*/ + +PyDoc_STRVAR(fn_with_default_binop_expr__doc__, +"fn_with_default_binop_expr($module, /, arg=a+b)\n" +"--\n" +"\n"); + +#define FN_WITH_DEFAULT_BINOP_EXPR_METHODDEF \ + {"fn_with_default_binop_expr", _PyCFunction_CAST(fn_with_default_binop_expr), METH_FASTCALL|METH_KEYWORDS, fn_with_default_binop_expr__doc__}, + +static PyObject * +fn_with_default_binop_expr_impl(PyObject *module, PyObject *arg); + +static PyObject * +fn_with_default_binop_expr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(arg), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"arg", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "fn_with_default_binop_expr", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *arg = CONST_A + CONST_B; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + arg = args[0]; +skip_optional_pos: + return_value = fn_with_default_binop_expr_impl(module, arg); + +exit: + return return_value; +} + +static PyObject * +fn_with_default_binop_expr_impl(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=018672772e4092ff input=1b55c8ae68d89453]*/ + +/*[python input] +class Custom_converter(CConverter): + type = "str" + default = "Hello!" + converter = "c_converter_func" +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=d612708f0efb8e3c]*/ + +/*[clinic input] +docstr_fallback_to_converter_default + a: Custom +Check docstring default value fallback. + +Verify that the docstring formatter fetches the default +value from the converter if no 'py_default' is found. +The signature should have the default a='Hello!', +as given by the Custom converter. +[clinic start generated code]*/ + +PyDoc_STRVAR(docstr_fallback_to_converter_default__doc__, +"docstr_fallback_to_converter_default($module, /, a=\'Hello!\')\n" +"--\n" +"\n" +"Check docstring default value fallback.\n" +"\n" +"Verify that the docstring formatter fetches the default\n" +"value from the converter if no \'py_default\' is found.\n" +"The signature should have the default a=\'Hello!\',\n" +"as given by the Custom converter."); + +#define DOCSTR_FALLBACK_TO_CONVERTER_DEFAULT_METHODDEF \ + {"docstr_fallback_to_converter_default", _PyCFunction_CAST(docstr_fallback_to_converter_default), METH_FASTCALL|METH_KEYWORDS, docstr_fallback_to_converter_default__doc__}, + +static PyObject * +docstr_fallback_to_converter_default_impl(PyObject *module, str a); + +static PyObject * +docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "docstr_fallback_to_converter_default", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + str a; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!c_converter_func(args[0], &a)) { + goto exit; + } + return_value = docstr_fallback_to_converter_default_impl(module, a); + +exit: + return return_value; +} + +static PyObject * +docstr_fallback_to_converter_default_impl(PyObject *module, str a) +/*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/ diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index fe3ee80b8d461f..b1ef332522d2ce 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import ALWAYS_EQ, NEVER_EQ, C_RECURSION_LIMIT class CommonTest(seq_tests.CommonTest): @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 613206a0855aea..5492bbf86d1f87 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -2,6 +2,7 @@ import unittest import collections import sys +from test.support import C_RECURSION_LIMIT class BasicTestMappingProtocol(unittest.TestCase): @@ -624,7 +625,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 3b332f49951f0c..64c66d8e25d9cd 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -6,7 +6,7 @@ import contextlib import functools import getpass -import opcode +import _opcode import os import re import stat @@ -64,7 +64,8 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", + "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT", + "skip_on_s390x", ] @@ -1092,7 +1093,7 @@ def requires_limited_api(test): def requires_specialization(test): return unittest.skipUnless( - opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) + _opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) def _filter_suite(suite, pred): """Recursively filter test cases in a suite based on a predicate.""" @@ -2460,3 +2461,10 @@ def adjust_int_max_str_digits(max_digits): #For recursion tests, easily exceeds default recursion limit EXCEEDS_RECURSION_LIMIT = 5000 + +# The default C recursion limit (from Include/cpython/pystate.h). +C_RECURSION_LIMIT = 1500 + +#Windows doesn't have os.uname() but it doesn't support s390x. +skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', + 'skipped on s390x') diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index a03fa4c7187b05..5346b39043f0f5 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1084,6 +1084,7 @@ def next(self): return self enum._test_simple_enum(_Precedence, ast._Precedence) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") @support.cpython_only def test_ast_recursion_limit(self): fail_depth = support.EXCEEDS_RECURSION_LIMIT diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 7f9dc621808358..9c92e75886c593 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -1074,6 +1074,29 @@ def test_eof_feed_when_closing_writer(self): self.assertEqual(messages, []) + def test_unclosed_resource_warnings(self): + async def inner(httpd): + rd, wr = await asyncio.open_connection(*httpd.address) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + data = await rd.readline() + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + data = await rd.read() + self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + with self.assertWarns(ResourceWarning): + del wr + gc.collect() + + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + with test_utils.run_test_server() as httpd: + self.loop.run_until_complete(inner(httpd)) + + self.assertEqual(messages, []) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 09a531f8cc627b..c3c3b1853b5736 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,5 +1,5 @@ import unittest -from test.support import cpython_only, requires_limited_api +from test.support import cpython_only, requires_limited_api, skip_on_s390x try: import _testcapi except ImportError: @@ -918,6 +918,7 @@ def test_multiple_values(self): @cpython_only class TestRecursion(unittest.TestCase): + @skip_on_s390x def test_super_deep(self): def recurse(n): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 0d3f93941e8205..e7cdd4be002a14 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2464,36 +2464,40 @@ def testfunc(x): def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" - def many_vars(): - # 260 vars, so z9 should have index 259 - a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 - b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 - c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 - d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 - e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 - f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 - g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 - h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 - i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 - j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 - k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 - l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 - m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 - n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 - o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 - p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 - q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 - r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 - s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 - t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 - u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 - v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 - w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 - x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 - y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 - z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 - while z9 > 0: - z9 = z9 - 1 + ns = {} + exec(textwrap.dedent(""" + def many_vars(): + # 260 vars, so z9 should have index 259 + a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 + b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 + c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 + d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 + e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 + f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 + g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 + h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 + i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 + j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 + k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 + l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 + m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 + n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 + o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 + p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 + q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 + s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 + t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 + u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 + v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 + w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 + x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 + y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 + z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 + while z9 > 0: + z9 = z9 - 1 + """), ns, ns) + many_vars = ns["many_vars"] opt = _testinternalcapi.get_uop_optimizer() with temporary_optimizer(opt): diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 3ce27d1dd6b487..f30fad2126940b 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -4,14 +4,14 @@ from test import support, test_tools from test.support import os_helper -from test.support import SHORT_TIMEOUT, requires_subprocess from test.support.os_helper import TESTFN, unlink from textwrap import dedent from unittest import TestCase import collections +import contextlib import inspect import os.path -import subprocess +import re import sys import unittest @@ -21,17 +21,24 @@ from clinic import DSLParser -class _ParserBase(TestCase): - maxDiff = None - - def expect_parser_failure(self, parser, _input): - with support.captured_stdout() as stdout: - with self.assertRaises(SystemExit): - parser(_input) - return stdout.getvalue() +def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): + """Helper for the parser tests. - def parse_function_should_fail(self, _input): - return self.expect_parser_failure(self.parse_function, _input) + tc: unittest.TestCase; passed self in the wrapper + parser: the clinic parser used for this test case + code: a str with input text (clinic code) + errmsg: the expected error message + filename: str, optional filename + lineno: int, optional line number + """ + code = dedent(code).strip() + errmsg = re.escape(errmsg) + with tc.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: + parser(code) + if filename is not None: + tc.assertEqual(cm.exception.filename, filename) + if lineno is not None: + tc.assertEqual(cm.exception.lineno, lineno) class FakeConverter: @@ -62,7 +69,7 @@ def __init__(self): self.converters = FakeConvertersDict() self.legacy_converters = FakeConvertersDict() self.language = clinic.CLanguage(None) - self.filename = None + self.filename = "clinic_tests" self.destination_buffers = {} self.block_parser = clinic.BlockParser('', self.language) self.modules = collections.OrderedDict() @@ -109,14 +116,16 @@ def __repr__(self): return "" -class ClinicWholeFileTest(_ParserBase): +class ClinicWholeFileTest(TestCase): + maxDiff = None + + def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): + _expect_failure(self, self.clinic.parse, raw, errmsg, + filename=filename, lineno=lineno) + def setUp(self): self.clinic = clinic.Clinic(clinic.CLanguage(None), filename="test.c") - def expect_failure(self, raw): - _input = dedent(raw).strip() - return self.expect_parser_failure(self.clinic.parse, _input) - def test_eol(self): # regression test: # clinic's block parser didn't recognize @@ -140,12 +149,11 @@ def test_mangled_marker_line(self): [clinic start generated code]*/ /*[clinic end generated code: foo]*/ """ - msg = ( - 'Error in file "test.c" on line 3:\n' - "Mangled Argument Clinic marker line: '/*[clinic end generated code: foo]*/'\n" + err = ( + "Mangled Argument Clinic marker line: " + "'/*[clinic end generated code: foo]*/'" ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + self.expect_failure(raw, err, filename="test.c", lineno=3) def test_checksum_mismatch(self): raw = """ @@ -153,38 +161,28 @@ def test_checksum_mismatch(self): [clinic start generated code]*/ /*[clinic end generated code: output=0123456789abcdef input=fedcba9876543210]*/ """ - msg = ( - 'Error in file "test.c" on line 3:\n' - 'Checksum mismatch!\n' - 'Expected: 0123456789abcdef\n' - 'Computed: da39a3ee5e6b4b0d\n' - ) - out = self.expect_failure(raw) - self.assertIn(msg, out) + err = ("Checksum mismatch! " + "Expected '0123456789abcdef', computed 'da39a3ee5e6b4b0d'") + self.expect_failure(raw, err, filename="test.c", lineno=3) def test_garbage_after_stop_line(self): raw = """ /*[clinic input] [clinic start generated code]*/foobarfoobar! """ - msg = ( - 'Error in file "test.c" on line 2:\n' - "Garbage after stop line: 'foobarfoobar!'\n" - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = "Garbage after stop line: 'foobarfoobar!'" + self.expect_failure(raw, err, filename="test.c", lineno=2) def test_whitespace_before_stop_line(self): raw = """ /*[clinic input] [clinic start generated code]*/ """ - msg = ( - 'Error in file "test.c" on line 2:\n' - "Whitespace is not allowed before the stop line: ' [clinic start generated code]*/'\n" + err = ( + "Whitespace is not allowed before the stop line: " + "' [clinic start generated code]*/'" ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + self.expect_failure(raw, err, filename="test.c", lineno=2) def test_parse_with_body_prefix(self): clang = clinic.CLanguage(None) @@ -214,12 +212,8 @@ def test_cpp_monitor_fail_nested_block_comment(self): */ */ """ - msg = ( - 'Error in file "test.c" on line 2:\n' - 'Nested block comment!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = 'Nested block comment!' + self.expect_failure(raw, err, filename="test.c", lineno=2) def test_cpp_monitor_fail_invalid_format_noarg(self): raw = """ @@ -227,12 +221,8 @@ def test_cpp_monitor_fail_invalid_format_noarg(self): a() #endif """ - msg = ( - 'Error in file "test.c" on line 1:\n' - 'Invalid format for #if line: no argument!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = 'Invalid format for #if line: no argument!' + self.expect_failure(raw, err, filename="test.c", lineno=1) def test_cpp_monitor_fail_invalid_format_toomanyargs(self): raw = """ @@ -240,39 +230,31 @@ def test_cpp_monitor_fail_invalid_format_toomanyargs(self): a() #endif """ - msg = ( - 'Error in file "test.c" on line 1:\n' - 'Invalid format for #ifdef line: should be exactly one argument!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = 'Invalid format for #ifdef line: should be exactly one argument!' + self.expect_failure(raw, err, filename="test.c", lineno=1) def test_cpp_monitor_fail_no_matching_if(self): raw = '#else' - msg = ( - 'Error in file "test.c" on line 1:\n' - '#else without matching #if / #ifdef / #ifndef!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = '#else without matching #if / #ifdef / #ifndef!' + self.expect_failure(raw, err, filename="test.c", lineno=1) def test_directive_output_unknown_preset(self): - out = self.expect_failure(""" + raw = """ /*[clinic input] output preset nosuchpreset [clinic start generated code]*/ - """) - msg = "Unknown preset 'nosuchpreset'" - self.assertIn(msg, out) + """ + err = "Unknown preset 'nosuchpreset'" + self.expect_failure(raw, err) def test_directive_output_cant_pop(self): - out = self.expect_failure(""" + raw = """ /*[clinic input] output pop [clinic start generated code]*/ - """) - msg = "Can't 'output pop', stack is empty" - self.assertIn(msg, out) + """ + err = "Can't 'output pop', stack is empty" + self.expect_failure(raw, err) def test_directive_output_print(self): raw = dedent(""" @@ -310,13 +292,381 @@ def test_directive_output_print(self): ) def test_unknown_destination_command(self): - out = self.expect_failure(""" + raw = """ /*[clinic input] destination buffer nosuchcommand [clinic start generated code]*/ + """ + err = "unknown destination command 'nosuchcommand'" + self.expect_failure(raw, err) + + def test_no_access_to_members_in_converter_init(self): + raw = """ + /*[python input] + class Custom_converter(CConverter): + converter = "some_c_function" + def converter_init(self): + self.function.noaccess + [python start generated code]*/ + /*[clinic input] + module test + test.fn + a: Custom + [clinic start generated code]*/ + """ + err = ( + "accessing self.function inside converter_init is disallowed!" + ) + self.expect_failure(raw, err) + + @staticmethod + @contextlib.contextmanager + def _clinic_version(new_version): + """Helper for test_version_*() tests""" + _saved = clinic.version + clinic.version = new_version + try: + yield + finally: + clinic.version = _saved + + def test_version_directive(self): + dataset = ( + # (clinic version, required version) + ('3', '2'), # required version < clinic version + ('3.1', '3.0'), # required version < clinic version + ('1.2b0', '1.2a7'), # required version < clinic version + ('5', '5'), # required version == clinic version + ('6.1', '6.1'), # required version == clinic version + ('1.2b3', '1.2b3'), # required version == clinic version + ) + for clinic_version, required_version in dataset: + with self.subTest(clinic_version=clinic_version, + required_version=required_version): + with self._clinic_version(clinic_version): + block = dedent(f""" + /*[clinic input] + version {required_version} + [clinic start generated code]*/ + """) + self.clinic.parse(block) + + def test_version_directive_insufficient_version(self): + with self._clinic_version('4'): + err = ( + "Insufficient Clinic version!\n" + " Version: 4\n" + " Required: 5" + ) + block = """ + /*[clinic input] + version 5 + [clinic start generated code]*/ + """ + self.expect_failure(block, err) + + def test_version_directive_illegal_char(self): + err = "Illegal character 'v' in version string 'v5'" + block = """ + /*[clinic input] + version v5 + [clinic start generated code]*/ + """ + self.expect_failure(block, err) + + def test_version_directive_unsupported_string(self): + err = "Unsupported version string: '.-'" + block = """ + /*[clinic input] + version .- + [clinic start generated code]*/ + """ + self.expect_failure(block, err) + + def test_clone_mismatch(self): + err = "'kind' of function and cloned function don't match!" + block = """ + /*[clinic input] + module m + @classmethod + m.f1 + a: object + [clinic start generated code]*/ + /*[clinic input] + @staticmethod + m.f2 = m.f1 + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=9) + + def test_badly_formed_return_annotation(self): + err = "Badly formed annotation for 'm.f': 'Custom'" + block = """ + /*[python input] + class Custom_return_converter(CReturnConverter): + def __init__(self): + raise ValueError("abc") + [python start generated code]*/ + /*[clinic input] + module m + m.f -> Custom + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=8) + + def test_module_already_got_one(self): + err = "Already defined module 'm'!" + block = """ + /*[clinic input] + module m + module m + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_destination_already_got_one(self): + err = "Destination already exists: 'test'" + block = """ + /*[clinic input] + destination test new buffer + destination test new buffer + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_destination_does_not_exist(self): + err = "Destination does not exist: '/dev/null'" + block = """ + /*[clinic input] + output everything /dev/null + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_class_already_got_one(self): + err = "Already defined class 'C'!" + block = """ + /*[clinic input] + class C "" "" + class C "" "" + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_cant_nest_module_inside_class(self): + err = "Can't nest a module inside a class!" + block = """ + /*[clinic input] + class C "" "" + module C.m + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_dest_buffer_not_empty_at_eof(self): + expected_warning = ("Destination buffer 'buffer' not empty at " + "end of file, emptying.") + expected_generated = dedent(""" + /*[clinic input] + output everything buffer + fn + a: object + / + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=1c4668687f5fd002]*/ + + /*[clinic input] + dump buffer + [clinic start generated code]*/ + + PyDoc_VAR(fn__doc__); + + PyDoc_STRVAR(fn__doc__, + "fn($module, a, /)\\n" + "--\\n" + "\\n"); + + #define FN_METHODDEF \\ + {"fn", (PyCFunction)fn, METH_O, fn__doc__}, + + static PyObject * + fn(PyObject *module, PyObject *a) + /*[clinic end generated code: output=be6798b148ab4e53 input=524ce2e021e4eba6]*/ """) - msg = "unknown destination command 'nosuchcommand'" - self.assertIn(msg, out) + block = dedent(""" + /*[clinic input] + output everything buffer + fn + a: object + / + [clinic start generated code]*/ + """) + with support.captured_stdout() as stdout: + generated = self.clinic.parse(block) + self.assertIn(expected_warning, stdout.getvalue()) + self.assertEqual(generated, expected_generated) + + def test_dest_clear(self): + err = "Can't clear destination 'file': it's not of type 'buffer'" + block = """ + /*[clinic input] + destination file clear + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_directive_set_misuse(self): + err = "unknown variable 'ets'" + block = """ + /*[clinic input] + set ets tse + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_directive_set_prefix(self): + block = dedent(""" + /*[clinic input] + set line_prefix '// ' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "// PyDoc_VAR(fn__doc__);" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_set_suffix(self): + block = dedent(""" + /*[clinic input] + set line_suffix ' // test' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "PyDoc_VAR(fn__doc__); // test" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_set_prefix_and_suffix(self): + block = dedent(""" + /*[clinic input] + set line_prefix '{block comment start} ' + set line_suffix ' {block comment end}' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "/* PyDoc_VAR(fn__doc__); */" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_printout(self): + block = dedent(""" + /*[clinic input] + output everything buffer + printout test + [clinic start generated code]*/ + """) + expected = dedent(""" + /*[clinic input] + output everything buffer + printout test + [clinic start generated code]*/ + test + /*[clinic end generated code: output=4e1243bd22c66e76 input=898f1a32965d44ca]*/ + """) + generated = self.clinic.parse(block) + self.assertEqual(generated, expected) + + def test_directive_preserve_twice(self): + err = "Can't have 'preserve' twice in one block!" + block = """ + /*[clinic input] + preserve + preserve + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_directive_preserve_input(self): + err = "'preserve' only works for blocks that don't produce any output!" + block = """ + /*[clinic input] + preserve + fn + a: object + / + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + + def test_directive_preserve_output(self): + err = "'preserve' only works for blocks that don't produce any output!" + block = dedent(""" + /*[clinic input] + output everything buffer + preserve + [clinic start generated code]*/ + // Preserve this + /*[clinic end generated code: output=eaa49677ae4c1f7d input=559b5db18fddae6a]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=524ce2e021e4eba6]*/ + """) + generated = self.clinic.parse(block) + self.assertEqual(generated, block) + + def test_directive_output_invalid_command(self): + err = dedent(""" + Invalid command or destination name 'cmd'. Must be one of: + - 'preset' + - 'push' + - 'pop' + - 'print' + - 'everything' + - 'cpp_if' + - 'docstring_prototype' + - 'docstring_definition' + - 'methoddef_define' + - 'impl_prototype' + - 'parser_prototype' + - 'parser_definition' + - 'cpp_endif' + - 'methoddef_ifndef' + - 'impl_definition' + """).strip() + block = """ + /*[clinic input] + output cmd buffer + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) class ClinicGroupPermuterTest(TestCase): @@ -495,7 +845,27 @@ def test_clinic_1(self): """) -class ClinicParserTest(_ParserBase): +class ClinicParserTest(TestCase): + + def parse(self, text): + c = FakeClinic() + parser = DSLParser(c) + block = clinic.Block(text) + parser.parse(block) + return block + + def parse_function(self, text, signatures_in_block=2, function_index=1): + block = self.parse(text) + s = block.signatures + self.assertEqual(len(s), signatures_in_block) + assert isinstance(s[0], clinic.Module) + assert isinstance(s[function_index], clinic.Function) + return s[function_index] + + def expect_failure(self, block, err, *, filename=None, lineno=None): + _expect_failure(self, self.parse_function, block, err, + filename=filename, lineno=lineno) + def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) self.assertEqual(fn.docstring.strip(), @@ -555,7 +925,7 @@ def test_param_with_continuations(self): p = function.parameters['follow_symlinks'] self.assertEqual(True, p.default) - def test_param_default_expression(self): + def test_param_default_expr_named_constant(self): function = self.parse_function(""" module os os.access @@ -565,17 +935,27 @@ def test_param_default_expression(self): self.assertEqual(sys.maxsize, p.default) self.assertEqual("MAXSIZE", p.converter.c_default) - expected_msg = ( - "Error on line 0:\n" - "When you specify a named constant ('sys.maxsize') as your default value,\n" - "you MUST specify a valid c_default.\n" + err = ( + "When you specify a named constant ('sys.maxsize') as your default value, " + "you MUST specify a valid c_default." ) - out = self.parse_function_should_fail(""" + block = """ module os os.access follow_symlinks: int = sys.maxsize - """) - self.assertEqual(out, expected_msg) + """ + self.expect_failure(block, err, lineno=2) + + def test_param_default_expr_binop(self): + err = ( + "When you specify an expression ('a + b') as your default value, " + "you MUST specify a valid c_default." + ) + block = """ + fn + follow_symlinks: int = a + b + """ + self.expect_failure(block, err, lineno=1) def test_param_no_docstring(self): function = self.parse_function(""" @@ -590,17 +970,17 @@ def test_param_no_docstring(self): self.assertIsInstance(conv, clinic.str_converter) def test_param_default_parameters_out_of_order(self): - expected_msg = ( - "Error on line 0:\n" - "Can't have a parameter without a default ('something_else')\n" - "after a parameter with a default!\n" + err = ( + "Can't have a parameter without a default ('something_else') " + "after a parameter with a default!" ) - out = self.parse_function_should_fail(""" + block = """ module os os.access follow_symlinks: bool = True - something_else: str""") - self.assertEqual(out, expected_msg) + something_else: str + """ + self.expect_failure(block, err, lineno=3) def disabled_test_converter_arguments(self): function = self.parse_function(""" @@ -631,6 +1011,28 @@ def test_function_docstring(self): Path to be examined """) + def test_docstring_trailing_whitespace(self): + function = self.parse_function( + "module t\n" + "t.s\n" + " a: object\n" + " Param docstring with trailing whitespace \n" + "Func docstring summary with trailing whitespace \n" + " \n" + "Func docstring body with trailing whitespace \n" + ) + self.checkDocstring(function, """ + s($module, /, a) + -- + + Func docstring summary with trailing whitespace + + a + Param docstring with trailing whitespace + + Func docstring body with trailing whitespace + """) + def test_explicit_parameters_in_docstring(self): function = self.parse_function(dedent(""" module foo @@ -655,6 +1057,38 @@ def test_explicit_parameters_in_docstring(self): Okay, we're done here. """) + def test_docstring_with_comments(self): + function = self.parse_function(dedent(""" + module foo + foo.bar + x: int + # We're about to have + # the documentation for x. + Documentation for x. + # We've just had + # the documentation for x. + y: int + + # We're about to have + # the documentation for foo. + This is the documentation for foo. + # We've just had + # the documentation for foo. + + Okay, we're done here. + """)) + self.checkDocstring(function, """ + bar($module, /, x, y) + -- + + This is the documentation for foo. + + x + Documentation for x. + + Okay, we're done here. + """) + def test_parser_regression_special_character_in_parameter_column_of_docstring_first_line(self): function = self.parse_function(dedent(""" module os @@ -677,17 +1111,19 @@ def test_c_name(self): self.assertEqual("os_stat_fn", function.c_basename) def test_cloning_nonexistent_function_correctly_fails(self): - stdout = self.parse_function_should_fail(""" - cloned = fooooooooooooooooooooooo + block = """ + cloned = fooooooooooooooooo This is trying to clone a nonexistent function!! + """ + err = "Couldn't find existing function 'fooooooooooooooooo'!" + with support.captured_stderr() as stderr: + self.expect_failure(block, err, lineno=0) + expected_debug_print = dedent("""\ + cls=None, module=, existing='fooooooooooooooooo' + (cls or module).functions=[] """) - expected_error = """\ -cls=None, module=, existing='fooooooooooooooooooooooo' -(cls or module).functions=[] -Error on line 0: -Couldn't find existing function 'fooooooooooooooooooooooo'! -""" - self.assertEqual(expected_error, stdout) + stderr = stderr.getvalue() + self.assertIn(expected_debug_print, stderr) def test_return_converter(self): function = self.parse_function(""" @@ -697,30 +1133,28 @@ def test_return_converter(self): self.assertIsInstance(function.return_converter, clinic.int_return_converter) def test_return_converter_invalid_syntax(self): - stdout = self.parse_function_should_fail(""" + block = """ module os os.stat -> invalid syntax - """) - expected_error = "Badly formed annotation for os.stat: 'invalid syntax'" - self.assertIn(expected_error, stdout) + """ + err = "Badly formed annotation for 'os.stat': 'invalid syntax'" + self.expect_failure(block, err) def test_legacy_converter_disallowed_in_return_annotation(self): - stdout = self.parse_function_should_fail(""" + block = """ module os os.stat -> "s" - """) - expected_error = "Legacy converter 's' not allowed as a return converter" - self.assertIn(expected_error, stdout) + """ + err = "Legacy converter 's' not allowed as a return converter" + self.expect_failure(block, err) def test_unknown_return_converter(self): - stdout = self.parse_function_should_fail(""" + block = """ module os - os.stat -> foooooooooooooooooooooooo - """) - expected_error = ( - "No available return converter called 'foooooooooooooooooooooooo'" - ) - self.assertIn(expected_error, stdout) + os.stat -> fooooooooooooooooooooooo + """ + err = "No available return converter called 'fooooooooooooooooooooooo'" + self.expect_failure(block, err) def test_star(self): function = self.parse_function(""" @@ -865,19 +1299,12 @@ def test_nested_groups(self): Attributes for the character. """) - def parse_function_should_fail(self, s): - with support.captured_stdout() as stdout: - with self.assertRaises(SystemExit): - self.parse_function(s) - return stdout.getvalue() - def test_disallowed_grouping__two_top_groups_on_left(self): - expected_msg = ( - 'Error on line 0:\n' - 'Function two_top_groups_on_left has an unsupported group ' - 'configuration. (Unexpected state 2.b)\n' + err = ( + "Function 'two_top_groups_on_left' has an unsupported group " + "configuration. (Unexpected state 2.b)" ) - out = self.parse_function_should_fail(""" + block = """ module foo foo.two_top_groups_on_left [ @@ -887,11 +1314,11 @@ def test_disallowed_grouping__two_top_groups_on_left(self): group2 : int ] param: int - """) - self.assertEqual(out, expected_msg) + """ + self.expect_failure(block, err, lineno=5) def test_disallowed_grouping__two_top_groups_on_right(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.two_top_groups_on_right param: int @@ -901,15 +1328,15 @@ def test_disallowed_grouping__two_top_groups_on_right(self): [ group2 : int ] - """) - msg = ( - "Function two_top_groups_on_right has an unsupported group " + """ + err = ( + "Function 'two_top_groups_on_right' has an unsupported group " "configuration. (Unexpected state 6.b)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__parameter_after_group_on_right(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.parameter_after_group_on_right param: int @@ -919,15 +1346,15 @@ def test_disallowed_grouping__parameter_after_group_on_right(self): ] group2 : int ] - """) - msg = ( + """ + err = ( "Function parameter_after_group_on_right has an unsupported group " "configuration. (Unexpected state 6.a)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__group_after_parameter_on_left(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.group_after_parameter_on_left [ @@ -937,15 +1364,15 @@ def test_disallowed_grouping__group_after_parameter_on_left(self): ] ] param: int - """) - msg = ( - "Function group_after_parameter_on_left has an unsupported group " + """ + err = ( + "Function 'group_after_parameter_on_left' has an unsupported group " "configuration. (Unexpected state 2.b)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__empty_group_on_left(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.empty_group [ @@ -954,15 +1381,15 @@ def test_disallowed_grouping__empty_group_on_left(self): group2 : int ] param: int - """) - msg = ( - "Function empty_group has an empty group.\n" + """ + err = ( + "Function 'empty_group' has an empty group. " "All groups must contain at least one parameter." ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__empty_group_on_right(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.empty_group param: int @@ -971,24 +1398,24 @@ def test_disallowed_grouping__empty_group_on_right(self): ] group2 : int ] - """) - msg = ( - "Function empty_group has an empty group.\n" + """ + err = ( + "Function 'empty_group' has an empty group. " "All groups must contain at least one parameter." ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__no_matching_bracket(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.empty_group param: int ] group2: int ] - """) - msg = "Function empty_group has a ] without a matching [." - self.assertIn(msg, out) + """ + err = "Function 'empty_group' has a ']' without a matching '['" + self.expect_failure(block, err) def test_no_parameters(self): function = self.parse_function(""" @@ -1017,31 +1444,32 @@ class foo.Bar "unused" "notneeded" self.assertEqual(1, len(function.parameters)) def test_illegal_module_line(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar => int / - """) - msg = "Illegal function name: foo.bar => int" - self.assertIn(msg, out) + """ + err = "Illegal function name: 'foo.bar => int'" + self.expect_failure(block, err) def test_illegal_c_basename(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar as 935 / - """) - msg = "Illegal C basename: 935" - self.assertIn(msg, out) + """ + err = "Illegal C basename: '935'" + self.expect_failure(block, err) def test_single_star(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar * * - """) - self.assertIn("Function bar uses '*' more than once.", out) + """ + err = "Function 'bar' uses '*' more than once." + self.expect_failure(block, err) def test_parameters_required_after_star(self): dataset = ( @@ -1050,39 +1478,38 @@ def test_parameters_required_after_star(self): "module foo\nfoo.bar\n this: int\n *", "module foo\nfoo.bar\n this: int\n *\nDocstring.", ) - msg = "Function bar specifies '*' without any parameters afterwards." + err = "Function 'bar' specifies '*' without any parameters afterwards." for block in dataset: with self.subTest(block=block): - out = self.parse_function_should_fail(block) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_single_slash(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar / / - """) - msg = ( - "Function bar has an unsupported group configuration. " + """ + err = ( + "Function 'bar' has an unsupported group configuration. " "(Unexpected state 0.d)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_double_slash(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar a: int / b: int / - """) - msg = "Function bar uses '/' more than once." - self.assertIn(msg, out) + """ + err = "Function 'bar' uses '/' more than once." + self.expect_failure(block, err) def test_mix_star_and_slash(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar x: int @@ -1090,38 +1517,35 @@ def test_mix_star_and_slash(self): * z: int / - """) - msg = ( - "Function bar mixes keyword-only and positional-only parameters, " + """ + err = ( + "Function 'bar' mixes keyword-only and positional-only parameters, " "which is unsupported." ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_parameters_not_permitted_after_slash_for_now(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar / x: int - """) - msg = ( - "Function bar has an unsupported group configuration. " + """ + err = ( + "Function 'bar' has an unsupported group configuration. " "(Unexpected state 0.d)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_parameters_no_more_than_one_vararg(self): - expected_msg = ( - "Error on line 0:\n" - "Too many var args\n" - ) - out = self.parse_function_should_fail(""" + err = "Too many var args" + block = """ module foo foo.bar *vararg1: object *vararg2: object - """) - self.assertEqual(out, expected_msg) + """ + self.expect_failure(block, err, lineno=0) def test_function_not_at_column_0(self): function = self.parse_function(""" @@ -1143,24 +1567,83 @@ def test_function_not_at_column_0(self): Nested docstring here, goeth. """) + def test_docstring_only_summary(self): + function = self.parse_function(""" + module m + m.f + summary + """) + self.checkDocstring(function, """ + f($module, /) + -- + + summary + """) + + def test_docstring_empty_lines(self): + function = self.parse_function(""" + module m + m.f + + + """) + self.checkDocstring(function, """ + f($module, /) + -- + """) + + def test_docstring_explicit_params_placement(self): + function = self.parse_function(""" + module m + m.f + a: int + Param docstring for 'a' will be included + b: int + c: int + Param docstring for 'c' will be included + This is the summary line. + + We'll now place the params section here: + {parameters} + And now for something completely different! + (Note the added newline) + """) + self.checkDocstring(function, """ + f($module, /, a, b, c) + -- + + This is the summary line. + + We'll now place the params section here: + a + Param docstring for 'a' will be included + c + Param docstring for 'c' will be included + + And now for something completely different! + (Note the added newline) + """) + def test_indent_stack_no_tabs(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar *vararg1: object \t*vararg2: object - """) - msg = "Tab characters are illegal in the Clinic DSL." - self.assertIn(msg, out) + """ + err = ("Tab characters are illegal in the Clinic DSL: " + r"'\t*vararg2: object'") + self.expect_failure(block, err) def test_indent_stack_illegal_outdent(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar a: object b: object - """) - self.assertIn("Illegal outdent", out) + """ + err = "Illegal outdent" + self.expect_failure(block, err) def test_directive(self): c = FakeClinic() @@ -1178,10 +1661,7 @@ def test_legacy_converters(self): self.assertIsInstance(conv, clinic.str_converter) def test_legacy_converters_non_string_constant_annotation(self): - expected_failure_message = ( - "Error on line 0:\n" - "Annotations must be either a name, a function call, or a string.\n" - ) + err = "Annotations must be either a name, a function call, or a string" dataset = ( 'module os\nos.access\n path: 42', 'module os\nos.access\n path: 42.42', @@ -1190,14 +1670,10 @@ def test_legacy_converters_non_string_constant_annotation(self): ) for block in dataset: with self.subTest(block=block): - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_failure_message) + self.expect_failure(block, err, lineno=2) def test_other_bizarre_things_in_annotations_fail(self): - expected_failure_message = ( - "Error on line 0:\n" - "Annotations must be either a name, a function call, or a string.\n" - ) + err = "Annotations must be either a name, a function call, or a string" dataset = ( 'module os\nos.access\n path: {"some": "dictionary"}', 'module os\nos.access\n path: ["list", "of", "strings"]', @@ -1205,30 +1681,24 @@ def test_other_bizarre_things_in_annotations_fail(self): ) for block in dataset: with self.subTest(block=block): - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_failure_message) + self.expect_failure(block, err, lineno=2) def test_kwarg_splats_disallowed_in_function_call_annotations(self): - expected_error_msg = ( - "Error on line 0:\n" - "Cannot use a kwarg splat in a function-call annotation\n" - ) + err = "Cannot use a kwarg splat in a function-call annotation" dataset = ( 'module fo\nfo.barbaz\n o: bool(**{None: "bang!"})', 'module fo\nfo.barbaz -> bool(**{None: "bang!"})', 'module fo\nfo.barbaz -> bool(**{"bang": 42})', 'module fo\nfo.barbaz\n o: bool(**{"bang": None})', ) - for fn in dataset: - with self.subTest(fn=fn): - out = self.parse_function_should_fail(fn) - self.assertEqual(out, expected_error_msg) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) def test_self_param_placement(self): - expected_error_msg = ( - "Error on line 0:\n" + err = ( "A 'self' parameter, if specified, must be the very first thing " - "in the parameter block.\n" + "in the parameter block." ) block = """ module foo @@ -1236,27 +1706,21 @@ def test_self_param_placement(self): a: int self: self(type="PyObject *") """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=3) def test_self_param_cannot_be_optional(self): - expected_error_msg = ( - "Error on line 0:\n" - "A 'self' parameter cannot be marked optional.\n" - ) + err = "A 'self' parameter cannot be marked optional." block = """ module foo foo.func self: self(type="PyObject *") = None """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=2) def test_defining_class_param_placement(self): - expected_error_msg = ( - "Error on line 0:\n" + err = ( "A 'defining_class' parameter, if specified, must either be the " - "first thing in the parameter block, or come just after 'self'.\n" + "first thing in the parameter block, or come just after 'self'." ) block = """ module foo @@ -1265,21 +1729,16 @@ def test_defining_class_param_placement(self): a: int cls: defining_class """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=4) def test_defining_class_param_cannot_be_optional(self): - expected_error_msg = ( - "Error on line 0:\n" - "A 'defining_class' parameter cannot be marked optional.\n" - ) + err = "A 'defining_class' parameter cannot be marked optional." block = """ module foo foo.func cls: defining_class(type="PyObject *") = None """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=2) def test_slot_methods_cannot_access_defining_class(self): block = """ @@ -1289,34 +1748,38 @@ class Foo "" "" cls: defining_class a: object """ - msg = "Slot methods cannot access their defining class." - with self.assertRaisesRegex(ValueError, msg): + err = "Slot methods cannot access their defining class." + with self.assertRaisesRegex(ValueError, err): self.parse_function(block) def test_new_must_be_a_class_method(self): - expected_error_msg = ( - "Error on line 0:\n" - "__new__ must be a class method!\n" - ) - out = self.parse_function_should_fail(""" + err = "__new__ must be a class method!" + block = """ module foo class Foo "" "" Foo.__new__ - """) - self.assertEqual(out, expected_error_msg) + """ + self.expect_failure(block, err, lineno=2) def test_init_must_be_a_normal_method(self): - expected_error_msg = ( - "Error on line 0:\n" - "__init__ must be a normal method, not a class or static method!\n" - ) - out = self.parse_function_should_fail(""" + err = "__init__ must be a normal method, not a class or static method!" + block = """ module foo class Foo "" "" @classmethod Foo.__init__ - """) - self.assertEqual(out, expected_error_msg) + """ + self.expect_failure(block, err, lineno=3) + + def test_duplicate_coexist(self): + err = "Called @coexist twice" + block = """ + module m + @coexist + @coexist + m.fn + """ + self.expect_failure(block, err, lineno=2) def test_unused_param(self): block = self.parse(""" @@ -1356,21 +1819,6 @@ def test_unused_param(self): parser_decl = p.simple_declaration(in_parser=True) self.assertNotIn("Py_UNUSED", parser_decl) - def parse(self, text): - c = FakeClinic() - parser = DSLParser(c) - block = clinic.Block(text) - parser.parse(block) - return block - - def parse_function(self, text, signatures_in_block=2, function_index=1): - block = self.parse(text) - s = block.signatures - self.assertEqual(len(s), signatures_in_block) - assert isinstance(s[0], clinic.Module) - assert isinstance(s[function_index], clinic.Function) - return s[function_index] - def test_scaffolding(self): # test repr on special values self.assertEqual(repr(clinic.unspecified), '') @@ -1382,39 +1830,169 @@ def test_scaffolding(self): 'The igloos are melting!\n' ) with support.captured_stdout() as stdout: - with self.assertRaises(SystemExit): - clinic.fail('The igloos are melting!', - filename='clown.txt', line_number=69) - actual = stdout.getvalue() - self.assertEqual(actual, expected) + errmsg = 'The igloos are melting' + with self.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: + clinic.fail(errmsg, filename='clown.txt', line_number=69) + exc = cm.exception + self.assertEqual(exc.filename, 'clown.txt') + self.assertEqual(exc.lineno, 69) + + def test_non_ascii_character_in_docstring(self): + block = """ + module test + test.fn + a: int + á param docstring + docstring fü bár baß + """ + with support.captured_stdout() as stdout: + self.parse(block) + # The line numbers are off; this is a known limitation. + expected = dedent("""\ + Warning in file 'clinic_tests' on line 0: + Non-ascii characters are not allowed in docstrings: 'á' + + Warning in file 'clinic_tests' on line 0: + Non-ascii characters are not allowed in docstrings: 'ü', 'á', 'ß' + + """) + self.assertEqual(stdout.getvalue(), expected) + + def test_illegal_c_identifier(self): + err = "Illegal C identifier: 17a" + block = """ + module test + test.fn + a as 17a: int + """ + self.expect_failure(block, err, lineno=2) + + def test_cannot_convert_special_method(self): + err = "__len__ is a special method and cannot be converted" + block = """ + class T "" "" + T.__len__ + """ + self.expect_failure(block, err, lineno=1) + + def test_cannot_specify_pydefault_without_default(self): + err = "You can't specify py_default without specifying a default value!" + block = """ + fn + a: object(py_default='NULL') + """ + self.expect_failure(block, err, lineno=1) + + def test_vararg_cannot_take_default_value(self): + err = "Vararg can't take a default value!" + block = """ + fn + *args: object = None + """ + self.expect_failure(block, err, lineno=1) + + def test_default_is_not_of_correct_type(self): + err = ("int_converter: default value 2.5 for field 'a' " + "is not of type 'int'") + block = """ + fn + a: int = 2.5 + """ + self.expect_failure(block, err, lineno=1) + + def test_invalid_legacy_converter(self): + err = "'fhi' is not a valid legacy converter" + block = """ + fn + a: 'fhi' + """ + self.expect_failure(block, err, lineno=1) + + def test_parent_class_or_module_does_not_exist(self): + err = "Parent class or module 'z' does not exist" + block = """ + module m + z.func + """ + self.expect_failure(block, err, lineno=1) + + def test_duplicate_param_name(self): + err = "You can't have two parameters named 'a'" + block = """ + module m + m.func + a: int + a: float + """ + self.expect_failure(block, err, lineno=3) + + def test_param_requires_custom_c_name(self): + err = "Parameter 'module' requires a custom C name" + block = """ + module m + m.func + module: int + """ + self.expect_failure(block, err, lineno=2) + + def test_state_func_docstring_assert_no_group(self): + err = "Function 'func' has a ']' without a matching '['" + block = """ + module m + m.func + ] + docstring + """ + self.expect_failure(block, err, lineno=2) + + def test_state_func_docstring_no_summary(self): + err = "Docstring for 'm.func' does not have a summary line!" + block = """ + module m + m.func + docstring1 + docstring2 + """ + self.expect_failure(block, err, lineno=0) + + def test_state_func_docstring_only_one_param_template(self): + err = "You may not specify {parameters} more than once in a docstring!" + block = """ + module m + m.func + docstring summary + + these are the params: + {parameters} + these are the params again: + {parameters} + """ + self.expect_failure(block, err, lineno=0) class ClinicExternalTest(TestCase): maxDiff = None - clinic_py = os.path.join(test_tools.toolsdir, "clinic", "clinic.py") - - def _do_test(self, *args, expect_success=True): - with subprocess.Popen( - [sys.executable, "-Xutf8", self.clinic_py, *args], - encoding="utf-8", - bufsize=0, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) as proc: - proc.wait() - if expect_success and proc.returncode: - self.fail("".join([*proc.stdout, *proc.stderr])) - stdout = proc.stdout.read() - stderr = proc.stderr.read() - # Clinic never writes to stderr. - self.assertEqual(stderr, "") - return stdout + + def run_clinic(self, *args): + with ( + support.captured_stdout() as out, + support.captured_stderr() as err, + self.assertRaises(SystemExit) as cm + ): + clinic.main(args) + return out.getvalue(), err.getvalue(), cm.exception.code def expect_success(self, *args): - return self._do_test(*args) + out, err, code = self.run_clinic(*args) + if code != 0: + self.fail("\n".join([f"Unexpected failure: {args=}", out, err])) + self.assertEqual(err, "") + return out def expect_failure(self, *args): - return self._do_test(*args, expect_success=False) + out, err, code = self.run_clinic(*args) + self.assertNotEqual(code, 0, f"Unexpected success: {args=}") + return out, err def test_external(self): CLINIC_TEST = 'clinic.test.c' @@ -1464,13 +2042,11 @@ def test_cli_force(self): const char *hand_edited = "output block is overwritten"; /*[clinic end generated code: output=bogus input=bogus]*/ """) - fail_msg = dedent(""" - Checksum mismatch! - Expected: bogus - Computed: 2ed19 - Suggested fix: remove all generated code including the end marker, - or use the '-f' option. - """) + fail_msg = ( + "Checksum mismatch! Expected 'bogus', computed '2ed19'. " + "Suggested fix: remove all generated code including the end marker, " + "or use the '-f' option.\n" + ) with os_helper.temp_dir() as tmp_dir: fn = os.path.join(tmp_dir, "test.c") with open(fn, "w", encoding="utf-8") as f: @@ -1478,8 +2054,9 @@ def test_cli_force(self): # First, run the CLI without -f and expect failure. # Note, we cannot check the entire fail msg, because the path to # the tmp file will change for every run. - out = self.expect_failure(fn) - self.assertTrue(out.endswith(fail_msg)) + _, err = self.expect_failure(fn) + self.assertTrue(err.endswith(fail_msg), + f"{err!r} does not end with {fail_msg!r}") # Then, force regeneration; success expected. out = self.expect_success("-f", fn) self.assertEqual(out, "") @@ -1621,34 +2198,92 @@ def test_cli_converters(self): ) def test_cli_fail_converters_and_filename(self): - out = self.expect_failure("--converters", "test.c") - msg = ( - "Usage error: can't specify --converters " - "and a filename at the same time" - ) - self.assertIn(msg, out) + _, err = self.expect_failure("--converters", "test.c") + msg = "can't specify --converters and a filename at the same time" + self.assertIn(msg, err) def test_cli_fail_no_filename(self): - out = self.expect_failure() - self.assertIn("usage: clinic.py", out) + _, err = self.expect_failure() + self.assertIn("no input files", err) def test_cli_fail_output_and_multiple_files(self): - out = self.expect_failure("-o", "out.c", "input.c", "moreinput.c") - msg = "Usage error: can't use -o with multiple filenames" - self.assertIn(msg, out) + _, err = self.expect_failure("-o", "out.c", "input.c", "moreinput.c") + msg = "error: can't use -o with multiple filenames" + self.assertIn(msg, err) def test_cli_fail_filename_or_output_and_make(self): + msg = "can't use -o or filenames with --make" for opts in ("-o", "out.c"), ("filename.c",): with self.subTest(opts=opts): - out = self.expect_failure("--make", *opts) - msg = "Usage error: can't use -o or filenames with --make" - self.assertIn(msg, out) + _, err = self.expect_failure("--make", *opts) + self.assertIn(msg, err) def test_cli_fail_make_without_srcdir(self): - out = self.expect_failure("--make", "--srcdir", "") - msg = "Usage error: --srcdir must not be empty with --make" - self.assertIn(msg, out) + _, err = self.expect_failure("--make", "--srcdir", "") + msg = "error: --srcdir must not be empty with --make" + self.assertIn(msg, err) + + def test_file_dest(self): + block = dedent(""" + /*[clinic input] + destination test new file {path}.h + output everything test + func + a: object + / + [clinic start generated code]*/ + """) + expected_checksum_line = ( + "/*[clinic end generated code: " + "output=da39a3ee5e6b4b0d input=b602ab8e173ac3bd]*/\n" + ) + expected_output = dedent("""\ + /*[clinic input] + preserve + [clinic start generated code]*/ + + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # include "pycore_gc.h" // PyGC_Head + # include "pycore_runtime.h" // _Py_ID() + #endif + + + PyDoc_VAR(func__doc__); + + PyDoc_STRVAR(func__doc__, + "func($module, a, /)\\n" + "--\\n" + "\\n"); + #define FUNC_METHODDEF \\ + {"func", (PyCFunction)func, METH_O, func__doc__}, + + static PyObject * + func(PyObject *module, PyObject *a) + /*[clinic end generated code: output=56c09670e89a0d9a input=a9049054013a1b77]*/ + """) + with os_helper.temp_dir() as tmp_dir: + in_fn = os.path.join(tmp_dir, "test.c") + out_fn = os.path.join(tmp_dir, "test.c.h") + with open(in_fn, "w", encoding="utf-8") as f: + f.write(block) + with open(out_fn, "w", encoding="utf-8") as f: + f.write("") # Write an empty output file! + # Clinic should complain about the empty output file. + _, err = self.expect_failure(in_fn) + expected_err = (f"Modified destination file {out_fn!r}; " + "not overwriting!") + self.assertIn(expected_err, err) + # Run clinic again, this time with the -f option. + out = self.expect_success("-f", in_fn) + # Read back the generated output. + with open(in_fn, encoding="utf-8") as f: + data = f.read() + expected_block = f"{block}{expected_checksum_line}" + self.assertEqual(data, expected_block) + with open(out_fn, encoding="utf-8") as f: + data = f.read() + self.assertEqual(data, expected_output) try: import _testclinic as ac_tester @@ -2104,8 +2739,11 @@ def test_gh_99233_refcount(self): self.assertEqual(arg_refcount_origin, arg_refcount_after) def test_gh_99240_double_free(self): - expected_error = r'gh_99240_double_free\(\) argument 2 must be encoded string without null bytes, not str' - with self.assertRaisesRegex(TypeError, expected_error): + err = re.escape( + "gh_99240_double_free() argument 2 must be encoded string " + "without null bytes, not str" + ) + with self.assertRaisesRegex(TypeError, err): ac_tester.gh_99240_double_free('a', '\0b') def test_cloned_func_exception_message(self): @@ -2392,5 +3030,93 @@ def test_suffix_all_lines(self): self.assertEqual(out, expected) +class ClinicReprTests(unittest.TestCase): + def test_Block_repr(self): + block = clinic.Block("foo") + expected_repr = "" + self.assertEqual(repr(block), expected_repr) + + block2 = clinic.Block("bar", "baz", [], "eggs", "spam") + expected_repr_2 = "" + self.assertEqual(repr(block2), expected_repr_2) + + block3 = clinic.Block( + input="longboi_" * 100, + dsl_name="wow_so_long", + signatures=[], + output="very_long_" * 100, + indent="" + ) + expected_repr_3 = ( + "" + ) + self.assertEqual(repr(block3), expected_repr_3) + + def test_Destination_repr(self): + destination = clinic.Destination( + "foo", type="file", clinic=FakeClinic(), args=("eggs",) + ) + self.assertEqual( + repr(destination), "" + ) + + destination2 = clinic.Destination("bar", type="buffer", clinic=FakeClinic()) + self.assertEqual(repr(destination2), "") + + def test_Module_repr(self): + module = clinic.Module("foo", FakeClinic()) + self.assertRegex(repr(module), r"") + + def test_Class_repr(self): + cls = clinic.Class("foo", FakeClinic(), None, 'some_typedef', 'some_type_object') + self.assertRegex(repr(cls), r"") + + def test_FunctionKind_repr(self): + self.assertEqual( + repr(clinic.FunctionKind.INVALID), "" + ) + self.assertEqual( + repr(clinic.FunctionKind.CLASS_METHOD), "" + ) + + def test_Function_and_Parameter_reprs(self): + function = clinic.Function( + name='foo', + module=FakeClinic(), + cls=None, + c_basename=None, + full_name='foofoo', + return_converter=clinic.init_return_converter(), + kind=clinic.FunctionKind.METHOD_INIT, + coexist=False + ) + self.assertEqual(repr(function), "") + + converter = clinic.self_converter('bar', 'bar', function) + parameter = clinic.Parameter( + "bar", + kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, + function=function, + converter=converter + ) + self.assertEqual(repr(parameter), "") + + def test_Monitor_repr(self): + monitor = clinic.cpp.Monitor() + self.assertRegex(repr(monitor), r"") + + monitor.line_number = 42 + monitor.stack.append(("token1", "condition1")) + self.assertRegex( + repr(monitor), r"" + ) + + monitor.stack.append(("token2", "condition2")) + self.assertRegex( + repr(monitor), + r"" + ) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 85ce0a4b39d854..770184c5ef9a91 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -11,10 +11,9 @@ import warnings from test import support from test.support import (script_helper, requires_debug_ranges, - requires_specialization) + requires_specialization, C_RECURSION_LIMIT) from test.support.os_helper import FakePath - class TestSpecifics(unittest.TestCase): def compile_single(self, source): @@ -112,7 +111,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = 2000 + repeat = int(C_RECURSION_LIMIT * 0.9) longexpr = 'x = x or ' + '-x' * repeat g = {} code = textwrap.dedent(''' @@ -558,16 +557,12 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is sys.getrecursionlimit() * the scaling factor - # in symtable.c (currently 3) - # We expect to fail *at* that limit, because we use up some of - # the stack depth limit in the test suite code - # So we check the expected limit and 75% of that - # XXX (ncoghlan): duplicating the scaling factor here is a little - # ugly. Perhaps it should be exposed somewhere... - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + # Expected limit is C_RECURSION_LIMIT * 2 + # Duplicating the limit here is a little ugly. + # Perhaps it should be exposed somewhere... + fail_depth = C_RECURSION_LIMIT * 2 + 1 + crash_depth = C_RECURSION_LIMIT * 100 + success_depth = int(C_RECURSION_LIMIT * 1.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index 39f70e864757d5..a1a8745e737fa2 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -192,7 +192,7 @@ class S8I(Structure): (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) def test_recursive_as_param(self): - class A(object): + class A: pass a = A() @@ -201,7 +201,7 @@ class A(object): c_int.from_param(a) -class AsParamWrapper(object): +class AsParamWrapper: def __init__(self, param): self._as_parameter_ = param @@ -209,7 +209,7 @@ class AsParamWrapperTestCase(BasicWrapTestCase): wrap = AsParamWrapper -class AsParamPropertyWrapper(object): +class AsParamPropertyWrapper: def __init__(self, param): self._param = param diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index 037677e37ab34a..6fe3e119672409 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -120,7 +120,7 @@ def test_unsupported_restype_2(self): def test_issue_7959(self): proto = self.functype.__func__(None) - class X(object): + class X: def func(self): pass def __init__(self): self.v = proto(self.func) diff --git a/Lib/test/test_ctypes/test_functions.py b/Lib/test/test_ctypes/test_functions.py index 9cf680f16620ac..08eecbc9ea4442 100644 --- a/Lib/test/test_ctypes/test_functions.py +++ b/Lib/test/test_ctypes/test_functions.py @@ -4,8 +4,8 @@ import unittest from ctypes import (CDLL, Structure, Array, CFUNCTYPE, byref, POINTER, pointer, ArgumentError, - c_char, c_wchar, c_byte, c_char_p, - c_short, c_int, c_long, c_longlong, + c_char, c_wchar, c_byte, c_char_p, c_wchar_p, + c_short, c_int, c_long, c_longlong, c_void_p, c_float, c_double, c_longdouble) from _ctypes import _Pointer, _SimpleCData @@ -92,6 +92,54 @@ def test_wchar_parm(self): "argument 2: TypeError: one character unicode string " "expected") + def test_c_char_p_parm(self): + """Test the error message when converting an incompatible type to c_char_p.""" + proto = CFUNCTYPE(c_int, c_char_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback(b"abc"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(10) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'int' object cannot be " + "interpreted as ctypes.c_char_p") + + def test_c_wchar_p_parm(self): + """Test the error message when converting an incompatible type to c_wchar_p.""" + proto = CFUNCTYPE(c_int, c_wchar_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback("abc"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(10) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'int' object cannot be " + "interpreted as ctypes.c_wchar_p") + + def test_c_void_p_parm(self): + """Test the error message when converting an incompatible type to c_void_p.""" + proto = CFUNCTYPE(c_int, c_void_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback(5), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(2.5) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'float' object cannot be " + "interpreted as ctypes.c_void_p") + def test_wchar_result(self): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] diff --git a/Lib/test/test_ctypes/test_keeprefs.py b/Lib/test/test_ctypes/test_keeprefs.py index 23b03b64b4a716..c6fe1de62eae7c 100644 --- a/Lib/test/test_ctypes/test_keeprefs.py +++ b/Lib/test/test_ctypes/test_keeprefs.py @@ -98,6 +98,33 @@ def test_p_cint(self): x = pointer(i) self.assertEqual(x._objects, {'1': i}) + def test_pp_ownership(self): + d = c_int(123) + n = c_int(456) + + p = pointer(d) + pp = pointer(p) + + self.assertIs(pp._objects['1'], p) + self.assertIs(pp._objects['0']['1'], d) + + pp.contents.contents = n + + self.assertIs(pp._objects['1'], p) + self.assertIs(pp._objects['0']['1'], n) + + self.assertIs(p._objects['1'], n) + self.assertEqual(len(p._objects), 1) + + del d + del p + + self.assertIs(pp._objects['0']['1'], n) + self.assertEqual(len(pp._objects), 2) + + del n + + self.assertEqual(len(pp._objects), 2) class PointerToStructure(unittest.TestCase): def test(self): diff --git a/Lib/test/test_ctypes/test_numbers.py b/Lib/test/test_ctypes/test_numbers.py index fd318f9a18e533..29108a28ec16e1 100644 --- a/Lib/test/test_ctypes/test_numbers.py +++ b/Lib/test/test_ctypes/test_numbers.py @@ -86,7 +86,7 @@ def test_byref(self): def test_floats(self): # c_float and c_double can be created from # Python int and float - class FloatLike(object): + class FloatLike: def __float__(self): return 2.0 f = FloatLike() @@ -97,15 +97,15 @@ def __float__(self): self.assertEqual(t(f).value, 2.0) def test_integers(self): - class FloatLike(object): + class FloatLike: def __float__(self): return 2.0 f = FloatLike() - class IntLike(object): + class IntLike: def __int__(self): return 2 d = IntLike() - class IndexLike(object): + class IndexLike: def __index__(self): return 2 i = IndexLike() diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index 40979212a627d8..d1eeee6b0306fe 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -157,7 +157,7 @@ def test_noctypes_argtype(self): # TypeError: has no from_param method self.assertRaises(TypeError, setattr, func, "argtypes", (object,)) - class Adapter(object): + class Adapter: def from_param(cls, obj): return None @@ -165,7 +165,7 @@ def from_param(cls, obj): self.assertEqual(func(None), None) self.assertEqual(func(object()), None) - class Adapter(object): + class Adapter: def from_param(cls, obj): return obj @@ -174,7 +174,7 @@ def from_param(cls, obj): self.assertRaises(ArgumentError, func, object()) self.assertEqual(func(c_void_p(42)), 42) - class Adapter(object): + class Adapter: def from_param(cls, obj): raise ValueError(obj) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index db67f37608f1f2..fc66a309788ac1 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -5701,6 +5701,36 @@ def test_c_disallow_instantiation(self): ContextManager = type(C.localcontext()) check_disallow_instantiation(self, ContextManager) + def test_c_signaldict_segfault(self): + # See gh-106263 for details. + SignalDict = type(C.Context().flags) + sd = SignalDict() + err_msg = "invalid signal dict" + + with self.assertRaisesRegex(ValueError, err_msg): + len(sd) + + with self.assertRaisesRegex(ValueError, err_msg): + iter(sd) + + with self.assertRaisesRegex(ValueError, err_msg): + repr(sd) + + with self.assertRaisesRegex(ValueError, err_msg): + sd[C.InvalidOperation] = True + + with self.assertRaisesRegex(ValueError, err_msg): + sd[C.InvalidOperation] + + with self.assertRaisesRegex(ValueError, err_msg): + sd == C.Context().flags + + with self.assertRaisesRegex(ValueError, err_msg): + C.Context().flags == sd + + with self.assertRaisesRegex(ValueError, err_msg): + sd.copy() + @requires_docstrings @requires_cdecimal class SignatureTest(unittest.TestCase): diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 79638340059f65..fbc6ce8282de3c 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper +from test.support import import_helper, C_RECURSION_LIMIT class DictTest(unittest.TestCase): @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 924f4a6829e19c..2bd9d6eef8cfc6 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -3,6 +3,7 @@ import pickle import sys import unittest +from test.support import C_RECURSION_LIMIT class DictSetTest(unittest.TestCase): @@ -279,7 +280,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index 4b971deacc1a5c..e1adf8e9748506 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -3,6 +3,7 @@ import re import subprocess import sys +import sysconfig import types import unittest @@ -173,6 +174,87 @@ class SystemTapOptimizedTests(TraceTests, unittest.TestCase): backend = SystemTapBackend() optimize_python = 2 +class CheckDtraceProbes(unittest.TestCase): + @classmethod + def setUpClass(cls): + if sysconfig.get_config_var('WITH_DTRACE'): + readelf_major_version, readelf_minor_version = cls.get_readelf_version() + if support.verbose: + print(f"readelf version: {readelf_major_version}.{readelf_minor_version}") + else: + raise unittest.SkipTest("CPython must be configured with the --with-dtrace option.") + + + @staticmethod + def get_readelf_version(): + try: + cmd = ["readelf", "--version"] + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + with proc: + version, stderr = proc.communicate() + + if proc.returncode: + raise Exception( + f"Command {' '.join(cmd)!r} failed " + f"with exit code {proc.returncode}: " + f"stdout={version!r} stderr={stderr!r}" + ) + except OSError: + raise unittest.SkipTest("Couldn't find readelf on the path") + + # Regex to parse: + # 'GNU readelf (GNU Binutils) 2.40.0\n' -> 2.40 + match = re.search(r"^(?:GNU) readelf.*?\b(\d+)\.(\d+)", version) + if match is None: + raise unittest.SkipTest(f"Unable to parse readelf version: {version}") + + return int(match.group(1)), int(match.group(2)) + + def get_readelf_output(self): + command = ["readelf", "-n", sys.executable] + stdout, _ = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ).communicate() + return stdout + + def test_check_probes(self): + readelf_output = self.get_readelf_output() + + available_probe_names = [ + "Name: import__find__load__done", + "Name: import__find__load__start", + "Name: audit", + "Name: gc__start", + "Name: gc__done", + ] + + for probe_name in available_probe_names: + with self.subTest(probe_name=probe_name): + self.assertIn(probe_name, readelf_output) + + @unittest.expectedFailure + def test_missing_probes(self): + readelf_output = self.get_readelf_output() + + # Missing probes will be added in the future. + missing_probe_names = [ + "Name: function__entry", + "Name: function__return", + "Name: line", + ] + + for probe_name in missing_probe_names: + with self.subTest(probe_name=probe_name): + self.assertIn(probe_name, readelf_output) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 2658e027ff3e2b..a02d54da35e948 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest - +from test.support import C_RECURSION_LIMIT class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(2000): + for i in range(C_RECURSION_LIMIT + 1): e = ExceptionGroup('eg', [e]) return e diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 9c23c2e82058c7..54378fced54699 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -6,9 +6,9 @@ test_tools.skip_if_missing('cases_generator') with test_tools.imports_under_tool('cases_generator'): + import generate_cases import analysis import formatting - import generate_cases from parsing import StackEffect @@ -46,28 +46,11 @@ def test_effect_sizes(self): (2, "(oparg<<1)"), ) - self.assertEqual( - formatting.string_effect_size( - formatting.list_effect_size(input_effects), - ), "1 + oparg + oparg*2", - ) - self.assertEqual( - formatting.string_effect_size( - formatting.list_effect_size(output_effects), - ), - "2 + oparg*4", - ) - self.assertEqual( - formatting.string_effect_size( - formatting.list_effect_size(other_effects), - ), - "2 + (oparg<<1)", - ) - class TestGeneratedCases(unittest.TestCase): def setUp(self) -> None: super().setUp() + self.maxDiff = None self.temp_dir = tempfile.gettempdir() self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") @@ -140,7 +123,8 @@ def test_inst_one_pop(self): """ output = """ TARGET(OP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; spam(); STACK_SHRINK(1); DISPATCH(); @@ -173,8 +157,9 @@ def test_inst_one_push_one_pop(self): """ output = """ TARGET(OP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; spam(); stack_pointer[-1] = res; DISPATCH(); @@ -190,9 +175,11 @@ def test_binary_op(self): """ output = """ TARGET(OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; spam(); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -209,9 +196,11 @@ def test_overlap(self): """ output = """ TARGET(OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *result; + right = stack_pointer[-1]; + left = stack_pointer[-2]; spam(); stack_pointer[-1] = result; DISPATCH(); @@ -235,8 +224,9 @@ def test_predictions_and_eval_breaker(self): } TARGET(OP3) { - PyObject *arg = stack_pointer[-1]; + PyObject *arg; PyObject *res; + arg = stack_pointer[-1]; DEOPT_IF(xxx, OP1); stack_pointer[-1] = res; CHECK_EVAL_BREAKER(); @@ -281,9 +271,11 @@ def test_error_if_pop(self): """ output = """ TARGET(OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; if (cond) goto pop_2_label; STACK_SHRINK(1); stack_pointer[-1] = res; @@ -299,7 +291,8 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; uint16_t counter = read_u16(&next_instr[0].cache); uint32_t extra = read_u32(&next_instr[1].cache); STACK_SHRINK(1); @@ -338,8 +331,10 @@ def test_macro_instruction(self): """ output = """ TARGET(OP1) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); op1(left, right); next_instr += 1; @@ -347,38 +342,38 @@ def test_macro_instruction(self): } TARGET(OP) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; - PyObject *_tmp_3 = stack_pointer[-3]; + static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *arg2; + PyObject *res; + // OP1 + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; uint16_t counter = read_u16(&next_instr[0].cache); op1(left, right); - _tmp_2 = left; - _tmp_1 = right; } + // OP2 + arg2 = stack_pointer[-3]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *arg2 = _tmp_3; - PyObject *res; uint32_t extra = read_u32(&next_instr[3].cache); res = op2(arg2, left, right); - _tmp_3 = res; } - next_instr += 5; - static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); STACK_SHRINK(2); - stack_pointer[-1] = _tmp_3; + stack_pointer[-1] = res; + next_instr += 5; DISPATCH(); } TARGET(OP3) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *arg2 = stack_pointer[-3]; + PyObject *right; + PyObject *left; + PyObject *arg2; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + arg2 = stack_pointer[-3]; res = op3(arg2, left, right); STACK_SHRINK(2); stack_pointer[-1] = res; @@ -396,9 +391,12 @@ def test_array_input(self): """ output = """ TARGET(OP) { - PyObject *above = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg*2)); - PyObject *below = stack_pointer[-(2 + oparg*2)]; + PyObject *above; + PyObject **values; + PyObject *below; + above = stack_pointer[-1]; + values = stack_pointer - 1 - oparg*2; + below = stack_pointer[-2 - oparg*2]; spam(); STACK_SHRINK(oparg*2); STACK_SHRINK(2); @@ -416,12 +414,13 @@ def test_array_output(self): output = """ TARGET(OP) { PyObject *below; - PyObject **values = stack_pointer - (2) + 1; + PyObject **values; PyObject *above; + values = stack_pointer - 1; spam(values, oparg); STACK_GROW(oparg*3); + stack_pointer[-2 - oparg*3] = below; stack_pointer[-1] = above; - stack_pointer[-(2 + oparg*3)] = below; DISPATCH(); } """ @@ -435,8 +434,9 @@ def test_array_input_output(self): """ output = """ TARGET(OP) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *above; + values = stack_pointer - oparg; spam(values, oparg); STACK_GROW(1); stack_pointer[-1] = above; @@ -453,8 +453,10 @@ def test_array_error_if(self): """ output = """ TARGET(OP) { - PyObject **values = (stack_pointer - oparg); - PyObject *extra = stack_pointer[-(1 + oparg)]; + PyObject **values; + PyObject *extra; + values = stack_pointer - oparg; + extra = stack_pointer[-1 - oparg]; if (oparg == 0) { STACK_SHRINK(oparg); goto pop_1_somewhere; } STACK_SHRINK(oparg); STACK_SHRINK(1); @@ -471,18 +473,21 @@ def test_cond_effect(self): """ output = """ TARGET(OP) { - PyObject *cc = stack_pointer[-1]; - PyObject *input = ((oparg & 1) == 1) ? stack_pointer[-(1 + (((oparg & 1) == 1) ? 1 : 0))] : NULL; - PyObject *aa = stack_pointer[-(2 + (((oparg & 1) == 1) ? 1 : 0))]; + PyObject *cc; + PyObject *input = NULL; + PyObject *aa; PyObject *xx; PyObject *output = NULL; PyObject *zz; + cc = stack_pointer[-1]; + if ((oparg & 1) == 1) { input = stack_pointer[-1 - ((oparg & 1) == 1 ? 1 : 0)]; } + aa = stack_pointer[-2 - ((oparg & 1) == 1 ? 1 : 0)]; output = spam(oparg, input); STACK_SHRINK((((oparg & 1) == 1) ? 1 : 0)); STACK_GROW(((oparg & 2) ? 1 : 0)); + stack_pointer[-2 - (oparg & 2 ? 1 : 0)] = xx; + if (oparg & 2) { stack_pointer[-1 - (oparg & 2 ? 1 : 0)] = output; } stack_pointer[-1] = zz; - if (oparg & 2) { stack_pointer[-(1 + ((oparg & 2) ? 1 : 0))] = output; } - stack_pointer[-(2 + ((oparg & 2) ? 1 : 0))] = xx; DISPATCH(); } """ @@ -500,29 +505,28 @@ def test_macro_cond_effect(self): """ output = """ TARGET(M) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; - PyObject *_tmp_3 = stack_pointer[-3]; + PyObject *right; + PyObject *middle; + PyObject *left; + PyObject *deep; + PyObject *extra = NULL; + PyObject *res; + // A + right = stack_pointer[-1]; + middle = stack_pointer[-2]; + left = stack_pointer[-3]; { - PyObject *right = _tmp_1; - PyObject *middle = _tmp_2; - PyObject *left = _tmp_3; # Body of A } + // B { - PyObject *deep; - PyObject *extra = NULL; - PyObject *res; # Body of B - _tmp_3 = deep; - if (oparg) { _tmp_2 = extra; } - _tmp_1 = res; } STACK_SHRINK(1); STACK_GROW((oparg ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg) { stack_pointer[-2] = _tmp_2; } - stack_pointer[-3] = _tmp_3; + stack_pointer[-2 - (oparg ? 1 : 0)] = deep; + if (oparg) { stack_pointer[-1 - (oparg ? 1 : 0)] = extra; } + stack_pointer[-1] = res; DISPATCH(); } """ diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 7a3fcc22be8d13..163ed824ff1fb0 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -150,6 +150,7 @@ def _ready_to_import(name=None, source=""): def restore__testsinglephase(*, _orig=_testsinglephase): # We started with the module imported and want to restore # it to its nominal state. + sys.modules.pop('_testsinglephase', None) _orig._clear_globals() _testinternalcapi.clear_extension('_testsinglephase', _orig.__file__) import _testsinglephase @@ -2125,7 +2126,7 @@ def clean_up(): _interpreters.run_string(interpid, textwrap.dedent(f''' name = {self.NAME!r} if name in sys.modules: - sys.modules[name]._clear_globals() + sys.modules.pop(name)._clear_globals() _testinternalcapi.clear_extension(name, {self.FILE!r}) ''')) _interpreters.destroy(interpid) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 3fbfc073255532..5c31748ce2caac 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -990,6 +990,9 @@ def f(self): with DirsOnSysPath(tempdir): import inspect_actual self.assertIn("correct", inspect.getsource(inspect_actual.A)) + # Remove the module from sys.modules to force it to be reloaded. + # This is necessary when the test is run multiple times. + sys.modules.pop("inspect_actual") @unittest.skipIf( support.is_emscripten or support.is_wasi, diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index a5388b2e5debd8..6f204948c9fc48 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1321,6 +1321,17 @@ def testGetIp(self): self.assertEqual(str(self.ipv6_scoped_interface.ip), '2001:658:22a:cafe:200::1') + def testIPv6IPv4MappedStringRepresentation(self): + long_prefix = '0000:0000:0000:0000:0000:ffff:' + short_prefix = '::ffff:' + ipv4 = '1.2.3.4' + ipv6_ipv4_str = short_prefix + ipv4 + ipv6_ipv4_addr = ipaddress.IPv6Address(ipv6_ipv4_str) + ipv6_ipv4_iface = ipaddress.IPv6Interface(ipv6_ipv4_str) + self.assertEqual(str(ipv6_ipv4_addr), ipv6_ipv4_str) + self.assertEqual(ipv6_ipv4_addr.exploded, long_prefix + ipv4) + self.assertEqual(str(ipv6_ipv4_iface.ip), ipv6_ipv4_str) + def testGetScopeId(self): self.assertEqual(self.ipv6_address.scope_id, None) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 5789a932c59037..74deec84336f72 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1903,11 +1903,11 @@ def _check(glob, expected): "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("dir*/**"), ["dirC/dirD"]) + _check(p.rglob("dir*/**/"), ["dirC/dirD"]) _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) _check(p.rglob("*/"), ["dirC/dirD"]) _check(p.rglob(""), ["dirC", "dirC/dirD"]) - _check(p.rglob("**"), ["dirC", "dirC/dirD"]) + _check(p.rglob("**/"), ["dirC", "dirC/dirD"]) # gh-91616, a re module regression _check(p.rglob("*.txt"), ["dirC/novel.txt"]) _check(p.rglob("*.*"), ["dirC/novel.txt"]) @@ -2057,7 +2057,20 @@ def test_glob_above_recursion_limit(self): path.mkdir(parents=True) with set_recursion_limit(recursion_limit): - list(base.glob('**')) + list(base.glob('**/')) + + def test_glob_recursive_no_trailing_slash(self): + P = self.cls + p = P(BASE) + with self.assertWarns(FutureWarning): + p.glob('**') + with self.assertWarns(FutureWarning): + p.glob('*/**') + with self.assertWarns(FutureWarning): + p.rglob('**') + with self.assertWarns(FutureWarning): + p.rglob('*/**') + def test_readlink(self): if not self.can_symlink: diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py index 2f8ba272164d32..d0b702d392cdf6 100644 --- a/Lib/test/test_tools/test_sundry.py +++ b/Lib/test/test_tools/test_sundry.py @@ -19,17 +19,11 @@ class TestSundryScripts(unittest.TestCase): # cleanly the logging module. @import_helper.mock_register_at_fork def test_sundry(self, mock_os): - old_modules = import_helper.modules_setup() - try: - for fn in os.listdir(scriptsdir): - if not fn.endswith('.py'): - continue - - name = fn[:-3] - import_tool(name) - finally: - # Unload all modules loaded in this test - import_helper.modules_cleanup(*old_modules) + for fn in os.listdir(scriptsdir): + if not fn.endswith('.py'): + continue + name = fn[:-3] + import_tool(name) if __name__ == '__main__': diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 81744940f25b82..f2efee90dc0240 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1397,6 +1397,7 @@ class A: pass class B(typing.Generic[T]): pass class C(B[int]): pass class D(B[str], float): pass + self.assertEqual(types.get_original_bases(A), (object,)) self.assertEqual(types.get_original_bases(B), (typing.Generic[T],)) self.assertEqual(types.get_original_bases(C), (B[int],)) @@ -1409,6 +1410,18 @@ class F(list[int]): pass self.assertEqual(types.get_original_bases(E), (list[T],)) self.assertEqual(types.get_original_bases(F), (list[int],)) + class FirstBase(typing.Generic[T]): pass + class SecondBase(typing.Generic[T]): pass + class First(FirstBase[int]): pass + class Second(SecondBase[int]): pass + class G(First, Second): pass + self.assertEqual(types.get_original_bases(G), (First, Second)) + + class First_(typing.Generic[T]): pass + class Second_(typing.Generic[T]): pass + class H(First_, Second_): pass + self.assertEqual(types.get_original_bases(H), (First_, Second_)) + class ClassBasedNamedTuple(typing.NamedTuple): x: int diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 3dac70eb12852c..2113757254c0ed 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -7,7 +7,7 @@ import pickle import random import sys -from test.support import bigmemtest, _1G, _4G +from test.support import bigmemtest, _1G, _4G, skip_on_s390x zlib = import_helper.import_module('zlib') @@ -44,10 +44,7 @@ # zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data # # Make the assumption that s390x always has an accelerator to simplify the skip -# condition. Windows doesn't have os.uname() but it doesn't support s390x. -skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', - 'skipped on s390x') - +# condition. class VersionTestCase(unittest.TestCase): diff --git a/Lib/types.py b/Lib/types.py index 6110e6e1de7249..b4aa19cec40c89 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -165,14 +165,11 @@ class Baz(list[str]): ... assert get_original_bases(int) == (object,) """ try: - return cls.__orig_bases__ + return cls.__dict__.get("__orig_bases__", cls.__bases__) except AttributeError: - try: - return cls.__bases__ - except AttributeError: - raise TypeError( - f'Expected an instance of type, not {type(cls).__name__!r}' - ) from None + raise TypeError( + f"Expected an instance of type, not {type(cls).__name__!r}" + ) from None class DynamicClassAttribute: diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 9729759434a9f4..0fc25ec74565df 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.1.1u", - url="https://www.openssl.org/source/openssl-1.1.1u.tar.gz", - checksum='e2f8d84b523eecd06c7be7626830370300fbcc15386bf5142d72758f6963ebc6', + name="OpenSSL 3.0.9", + url="https://www.openssl.org/source/openssl-3.0.9.tar.gz", + checksum='eb1ab04781474360f77c318ab89d8c5a03abc38e63d65a603cabbf1b00a1dc90', buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index 5bc356d5267045..efd76b9b1ae64b 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -11,7 +11,7 @@ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \f1\b \cf0 NOTE: -\f0\b0 This is a beta preview of Python 3.12.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is an alpha preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \cf0 \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 @@ -19,14 +19,14 @@ \f1\b \cf0 \ul \ulc0 Certificate verification and OpenSSL\ \f0\b0 \ulnone \ -This package includes its own private copy of OpenSSL 1.1.1. The trust certificates in system and user keychains managed by the +This package includes its own private copy of OpenSSL 3.0. The trust certificates in system and user keychains managed by the \f2\i Keychain Access \f0\i0 application and the \f2\i security \f0\i0 command line utility are not used as defaults by the Python \f3 ssl \f0 module. A sample command script is included in -\f3 /Applications/Python 3.11 +\f3 /Applications/Python 3.13 \f0 to install a curated bundle of default root certificates from the third-party \f3 certifi \f0 package ({\field{\*\fldinst{HYPERLINK "https://pypi.org/project/certifi/"}}{\fldrslt https://pypi.org/project/certifi/}}). Double-click on diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 83b7aa9d883a16..79851e1f4a69cc 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -12,9 +12,8 @@ \f1\b macOS $MACOSX_DEPLOYMENT_TARGET \f0\b0 .\ \ -\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 -\f1\b \cf0 Python for macOS +\f1\b Python for macOS \f0\b0 consists of the {\field{\*\fldinst{HYPERLINK "https://www.python.org"}}{\fldrslt Python}} programming language interpreter and its batteries-included standard library to allow easy access to macOS features. It also includes the Python integrated development environment, \f1\b IDLE \f0\b0 . You can also use the included @@ -27,5 +26,5 @@ At the end of this install, click on \ \f1\b NOTE: -\f0\b0 This is a beta test preview of Python 3.12.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is an alpha test preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ } \ No newline at end of file diff --git a/Makefile.pre.in b/Makefile.pre.in index 3725feaca66ce3..12409774746a30 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1316,7 +1316,7 @@ regen-limited-abi: all # Regenerate all generated files .PHONY: regen-all -regen-all: regen-cases regen-opcode regen-opcode-targets regen-typeslots \ +regen-all: regen-cases regen-opcode regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen clinic \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ regen-test-levenshtein regen-global-objects @@ -1428,8 +1428,10 @@ regen-opcode: $(srcdir)/Lib/opcode.py \ $(srcdir)/Lib/_opcode_metadata.py \ $(srcdir)/Include/opcode.h.new \ + $(srcdir)/Python/opcode_targets.h.new \ $(srcdir)/Include/internal/pycore_opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new + $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new .PHONY: regen-token @@ -1531,13 +1533,6 @@ Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c $(UNICODE_DEPS) Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h -.PHONY: regen-opcode-targets -regen-opcode-targets: - # Regenerate Python/opcode_targets.h from Lib/opcode.py - # using Python/makeopcodetargets.py - $(PYTHON_FOR_REGEN) $(srcdir)/Python/makeopcodetargets.py \ - $(srcdir)/Python/opcode_targets.h.new - $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new .PHONY: regen-cases regen-cases: diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 40f6a6cb430d35..5178f4055e7b8e 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -2752,7 +2752,7 @@ by Shin-myoung-serp. .. section: Library Add deprecation warning for enum ``member.member`` access (e.g. -``Color.RED.BLUE``). +``Color.RED.BLUE``). Remove ``EnumMeta.__getattr__``. .. diff --git a/Misc/NEWS.d/next/Build/2023-08-01-17-12-53.gh-issue-105481.42nsDE.rst b/Misc/NEWS.d/next/Build/2023-08-01-17-12-53.gh-issue-105481.42nsDE.rst new file mode 100644 index 00000000000000..1e61c37b609469 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-08-01-17-12-53.gh-issue-105481.42nsDE.rst @@ -0,0 +1 @@ +Remove the make target ``regen-opcode-targets``, merge its work into ``regen-opcode`` which repeats most of the calculation. This simplifies the code for the build and reduces code duplication. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst new file mode 100644 index 00000000000000..7fb5b45c763e45 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst @@ -0,0 +1,2 @@ +Fix a segmentation fault caused by a use-after-free bug in ``frame_dealloc`` +when the trashcan delays the deallocation of a ``PyFrameObject``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-29-22-01-30.gh-issue-104584.tINuoA.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-29-22-01-30.gh-issue-104584.tINuoA.rst new file mode 100644 index 00000000000000..059524831597b7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-29-22-01-30.gh-issue-104584.tINuoA.rst @@ -0,0 +1,2 @@ +Fix an issue which caused incorrect inline caches to be read when running +with :envvar:`PYTHONUOPS` or :option:`-X uops <-X>` enabled. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst new file mode 100644 index 00000000000000..fb0940b456dae5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst @@ -0,0 +1,3 @@ +Increase C recursion limit for functions other than the main interpreter +from 800 to 1500. This should allow functions like ``list.__repr__`` and +``json.dumps`` to handle all the inputs that they could prior to 3.12 diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-14-18-49.gh-issue-107455.Es53l7.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-14-18-49.gh-issue-107455.Es53l7.rst new file mode 100644 index 00000000000000..84a93251e799d5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-14-18-49.gh-issue-107455.Es53l7.rst @@ -0,0 +1,3 @@ +Improve error messages when converting an incompatible type to +:class:`ctypes.c_char_p`, :class:`ctypes.c_wchar_p` and +:class:`ctypes.c_void_p`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst new file mode 100644 index 00000000000000..99ebc926e2ce2d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst @@ -0,0 +1,2 @@ +Clear generators' exception state after ``return`` to break reference +cycles. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-01-09-41-36.gh-issue-106608.OFZogw.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-01-09-41-36.gh-issue-106608.OFZogw.rst new file mode 100644 index 00000000000000..20d43a7c4f754a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-01-09-41-36.gh-issue-106608.OFZogw.rst @@ -0,0 +1 @@ +Make ``_PyUOpExecutorObject`` variable length. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst new file mode 100644 index 00000000000000..5084c854360e35 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst @@ -0,0 +1,4 @@ +Trace refs builds (``--with-trace-refs``) were crashing when used with +isolated subinterpreters. The problematic global state has been isolated to +each interpreter. Other fixing the crashes, this change does not affect +users. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-03-13-38-14.gh-issue-84436.gl1wHx.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-03-13-38-14.gh-issue-84436.gl1wHx.rst new file mode 100644 index 00000000000000..71044c32feebcc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-03-13-38-14.gh-issue-84436.gl1wHx.rst @@ -0,0 +1 @@ +Skip reference count modifications for many known immortal objects. diff --git a/Misc/NEWS.d/next/Library/2021-08-16-17-52-26.bpo-44850.r8jx5u.rst b/Misc/NEWS.d/next/Library/2021-08-16-17-52-26.bpo-44850.r8jx5u.rst new file mode 100644 index 00000000000000..1fe5497f856e90 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-16-17-52-26.bpo-44850.r8jx5u.rst @@ -0,0 +1,2 @@ +Improve performance of :func:`operator.methodcaller` using the :pep:`590` ``vectorcall`` convention. +Patch by Anthony Lee and Pieter Eendebak. diff --git a/Misc/NEWS.d/next/Library/2021-10-31-16-06-28.bpo-43633.vflwXv.rst b/Misc/NEWS.d/next/Library/2021-10-31-16-06-28.bpo-43633.vflwXv.rst new file mode 100644 index 00000000000000..025de1e1a7d6ef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-31-16-06-28.bpo-43633.vflwXv.rst @@ -0,0 +1 @@ +Improve the textual representation of IPv4-mapped IPv6 addresses (:rfc:`4291` Sections 2.2, 2.5.5.2) in :mod:`ipaddress`. Patch by Oleksandr Pavliuk. diff --git a/Misc/NEWS.d/next/Library/2023-06-07-00-13-00.gh-issue-70303.frwUKH.rst b/Misc/NEWS.d/next/Library/2023-06-07-00-13-00.gh-issue-70303.frwUKH.rst new file mode 100644 index 00000000000000..39a891ac5964ab --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-07-00-13-00.gh-issue-70303.frwUKH.rst @@ -0,0 +1,4 @@ +Emit :exc:`FutureWarning` from :meth:`pathlib.Path.glob` and +:meth:`~pathlib.Path.rglob` if the given pattern ends with "``**``". In a +future Python release, patterns with this ending will match both files and +directories. Add a trailing slash to only match directories. diff --git a/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst b/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst new file mode 100644 index 00000000000000..23763818d84ba5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst @@ -0,0 +1,2 @@ +Fix crash when calling ``repr`` with a manually constructed SignalDict object. +Patch by Charlie Zhao. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2023-07-23-13-05-32.gh-issue-105578.XAQtyR.rst b/Misc/NEWS.d/next/Library/2023-07-23-13-05-32.gh-issue-105578.XAQtyR.rst new file mode 100644 index 00000000000000..4a03f5c35ff6c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-23-13-05-32.gh-issue-105578.XAQtyR.rst @@ -0,0 +1,2 @@ +Deprecate :class:`typing.AnyStr` in favor of the new Type Parameter syntax. +See PEP 695. diff --git a/Misc/NEWS.d/next/Library/2023-07-24-01-21-16.gh-issue-46376.w-xuDL.rst b/Misc/NEWS.d/next/Library/2023-07-24-01-21-16.gh-issue-46376.w-xuDL.rst new file mode 100644 index 00000000000000..8e8f0245b4539b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-24-01-21-16.gh-issue-46376.w-xuDL.rst @@ -0,0 +1 @@ +Prevent memory leak and use-after-free when using pointers to pointers with ctypes diff --git a/Misc/NEWS.d/next/Library/2023-08-01-15-17-20.gh-issue-105481.vMbmj_.rst b/Misc/NEWS.d/next/Library/2023-08-01-15-17-20.gh-issue-105481.vMbmj_.rst new file mode 100644 index 00000000000000..153c18a6f00953 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-01-15-17-20.gh-issue-105481.vMbmj_.rst @@ -0,0 +1 @@ +:data:`opcode.ENABLE_SPECIALIZATION` (which was added in 3.12 but never documented or intended for external usage) is moved to :data:`_opcode.ENABLE_SPECIALIZATION` where tests can access it. diff --git a/Misc/NEWS.d/next/Library/2023-08-01-21-43-58.gh-issue-105481.cl2ajS.rst b/Misc/NEWS.d/next/Library/2023-08-01-21-43-58.gh-issue-105481.cl2ajS.rst new file mode 100644 index 00000000000000..d02f909e870188 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-01-21-43-58.gh-issue-105481.cl2ajS.rst @@ -0,0 +1,2 @@ +Remove ``opcode.is_pseudo``, ``opcode.MIN_PSEUDO_OPCODE`` and ``opcode.MAX_PSEUDO_OPCODE``, +which were added in 3.12, were never documented and were not intended to be used externally. diff --git a/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst b/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst new file mode 100644 index 00000000000000..67677dd3c8ed24 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst @@ -0,0 +1,3 @@ +Fix :func:`types.get_original_bases` to only return +:attr:`!__orig_bases__` if it is present on ``cls`` directly. Patch by +James Hilton-Balfe. diff --git a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst new file mode 100644 index 00000000000000..ecaf437a48e0ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst @@ -0,0 +1,6 @@ +Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` +instead of ``SSL_ERROR_SSL`` when a certification verification has failed, +but the error parameters will still contain ``ERR_LIB_SSL`` and +``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and +raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo +Galindo diff --git a/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst b/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst new file mode 100644 index 00000000000000..02c52d714e9df7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst @@ -0,0 +1 @@ +Raise :exc:`ResourceWarning` when :class:`asyncio.StreamWriter` is not closed leading to memory leaks. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-07-30-23-32-16.gh-issue-107467.5O9p3G.rst b/Misc/NEWS.d/next/Tools-Demos/2023-07-30-23-32-16.gh-issue-107467.5O9p3G.rst new file mode 100644 index 00000000000000..2996837371be0f --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-07-30-23-32-16.gh-issue-107467.5O9p3G.rst @@ -0,0 +1,2 @@ +The Argument Clinic command-line tool now prints to stderr instead of stdout +on failure. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst new file mode 100644 index 00000000000000..080a6c15d9b8c5 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst @@ -0,0 +1,3 @@ +Fix duplicate module check in Argument Clinic. Previously, a duplicate +definition would incorrectly be silently accepted. Patch by Erlend E. +Aasland. diff --git a/Misc/NEWS.d/next/macOS/2023-07-30-23-42-20.gh-issue-99079.JAtoh1.rst b/Misc/NEWS.d/next/macOS/2023-07-30-23-42-20.gh-issue-99079.JAtoh1.rst new file mode 100644 index 00000000000000..d0eef4ec1003ce --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-07-30-23-42-20.gh-issue-99079.JAtoh1.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 3.0.9. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index f5a589b00c48d0..39c803355ba95b 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1398,7 +1398,8 @@ FutureObj_get_state(FutureObj *fut, void *Py_UNUSED(ignored)) default: assert (0); } - return Py_XNewRef(ret); + assert(_Py_IsImmortal(ret)); + return ret; } static PyObject * diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index c20d6ae55a06e7..9aee37a9d954ef 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1728,9 +1728,9 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) Py_DECREF(as_parameter); return value; } - /* XXX better message */ - PyErr_SetString(PyExc_TypeError, - "wrong type"); + PyErr_Format(PyExc_TypeError, + "'%.200s' object cannot be interpreted " + "as ctypes.c_wchar_p", Py_TYPE(value)->tp_name); return NULL; } @@ -1792,9 +1792,9 @@ c_char_p_from_param(PyObject *type, PyObject *value) Py_DECREF(as_parameter); return value; } - /* XXX better message */ - PyErr_SetString(PyExc_TypeError, - "wrong type"); + PyErr_Format(PyExc_TypeError, + "'%.200s' object cannot be interpreted " + "as ctypes.c_char_p", Py_TYPE(value)->tp_name); return NULL; } @@ -1927,9 +1927,9 @@ c_void_p_from_param(PyObject *type, PyObject *value) Py_DECREF(as_parameter); return value; } - /* XXX better message */ - PyErr_SetString(PyExc_TypeError, - "wrong type"); + PyErr_Format(PyExc_TypeError, + "'%.200s' object cannot be interpreted " + "as ctypes.c_void_p", Py_TYPE(value)->tp_name); return NULL; } @@ -5129,6 +5129,8 @@ static PyObject * Pointer_get_contents(CDataObject *self, void *closure) { StgDictObject *stgdict; + PyObject *keep, *ptr_probe; + CDataObject *ptr2ptr; if (*(void **)self->b_ptr == NULL) { PyErr_SetString(PyExc_ValueError, @@ -5138,6 +5140,33 @@ Pointer_get_contents(CDataObject *self, void *closure) stgdict = PyObject_stgdict((PyObject *)self); assert(stgdict); /* Cannot be NULL for pointer instances */ + + keep = GetKeepedObjects(self); + if (keep != NULL) { + // check if it's a pointer to a pointer: + // pointers will have '0' key in the _objects + ptr_probe = PyDict_GetItemString(keep, "0"); + + if (ptr_probe != NULL) { + ptr2ptr = (CDataObject*) PyDict_GetItemString(keep, "1"); + if (ptr2ptr == NULL) { + PyErr_SetString(PyExc_ValueError, + "Unexpected NULL pointer in _objects"); + return NULL; + } + // don't construct a new object, + // return existing one instead to preserve refcount + assert( + *(void**) self->b_ptr == ptr2ptr->b_ptr || + *(void**) self->b_value.c == ptr2ptr->b_ptr || + *(void**) self->b_ptr == ptr2ptr->b_value.c || + *(void**) self->b_value.c == ptr2ptr->b_value.c + ); // double-check that we are returning the same thing + Py_INCREF(ptr2ptr); + return (PyObject *) ptr2ptr; + } + } + return PyCData_FromBaseObj(stgdict->proto, (PyObject *)self, 0, *(void **)self->b_ptr); diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 7a206973975cc8..585214cc45d6cd 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -314,14 +314,12 @@ value_error_int(const char *mesg) return -1; } -#ifdef CONFIG_32 static PyObject * value_error_ptr(const char *mesg) { PyErr_SetString(PyExc_ValueError, mesg); return NULL; } -#endif static int type_error_int(const char *mesg) @@ -608,6 +606,8 @@ getround(decimal_state *state, PyObject *v) initialized to new SignalDicts. Once a SignalDict is tied to a context, it cannot be deleted. */ +static const char *INVALID_SIGNALDICT_ERROR_MSG = "invalid signal dict"; + static int signaldict_init(PyObject *self, PyObject *args UNUSED, PyObject *kwds UNUSED) { @@ -616,14 +616,20 @@ signaldict_init(PyObject *self, PyObject *args UNUSED, PyObject *kwds UNUSED) } static Py_ssize_t -signaldict_len(PyObject *self UNUSED) +signaldict_len(PyObject *self) { + if (SdFlagAddr(self) == NULL) { + return value_error_int(INVALID_SIGNALDICT_ERROR_MSG); + } return SIGNAL_MAP_LEN; } static PyObject * signaldict_iter(PyObject *self) { + if (SdFlagAddr(self) == NULL) { + return value_error_ptr(INVALID_SIGNALDICT_ERROR_MSG); + } decimal_state *state = get_module_state_by_def(Py_TYPE(self)); return PyTuple_Type.tp_iter(state->SignalTuple); } @@ -632,6 +638,9 @@ static PyObject * signaldict_getitem(PyObject *self, PyObject *key) { uint32_t flag; + if (SdFlagAddr(self) == NULL) { + return value_error_ptr(INVALID_SIGNALDICT_ERROR_MSG); + } decimal_state *state = get_module_state_by_def(Py_TYPE(self)); flag = exception_as_flag(state, key); @@ -648,11 +657,15 @@ signaldict_setitem(PyObject *self, PyObject *key, PyObject *value) uint32_t flag; int x; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + if (SdFlagAddr(self) == NULL) { + return value_error_int(INVALID_SIGNALDICT_ERROR_MSG); + } + if (value == NULL) { return value_error_int("signal keys cannot be deleted"); } + decimal_state *state = get_module_state_by_def(Py_TYPE(self)); flag = exception_as_flag(state, key); if (flag & DEC_ERRORS) { return -1; @@ -697,6 +710,10 @@ signaldict_repr(PyObject *self) const char *b[SIGNAL_MAP_LEN]; /* bool */ int i; + if (SdFlagAddr(self) == NULL) { + return value_error_ptr(INVALID_SIGNALDICT_ERROR_MSG); + } + assert(SIGNAL_MAP_LEN == 9); decimal_state *state = get_module_state_by_def(Py_TYPE(self)); @@ -721,6 +738,10 @@ signaldict_richcompare(PyObject *v, PyObject *w, int op) decimal_state *state = find_state_left_or_right(v, w); assert(PyDecSignalDict_Check(state, v)); + if ((SdFlagAddr(v) == NULL) || (SdFlagAddr(w) == NULL)) { + return value_error_ptr(INVALID_SIGNALDICT_ERROR_MSG); + } + if (op == Py_EQ || op == Py_NE) { if (PyDecSignalDict_Check(state, w)) { res = (SdFlags(v)==SdFlags(w)) ^ (op==Py_NE) ? Py_True : Py_False; @@ -748,6 +769,9 @@ signaldict_richcompare(PyObject *v, PyObject *w, int op) static PyObject * signaldict_copy(PyObject *self, PyObject *args UNUSED) { + if (SdFlagAddr(self) == NULL) { + return value_error_ptr(INVALID_SIGNALDICT_ERROR_MSG); + } decimal_state *state = get_module_state_by_def(Py_TYPE(self)); return flags_as_dict(state, SdFlags(self)); } diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 24d846e6634375..6ce90b2ed774c0 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -234,7 +234,7 @@ _io_IncrementalNewlineDecoder___init___impl(nldecoder_object *self, { if (errors == NULL) { - errors = Py_NewRef(&_Py_ID(strict)); + errors = &_Py_ID(strict); } else { errors = Py_NewRef(errors); @@ -1138,7 +1138,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, if (encoding == NULL && _PyRuntime.preconfig.utf8_mode) { _Py_DECLARE_STR(utf_8, "utf-8"); - self->encoding = Py_NewRef(&_Py_STR(utf_8)); + self->encoding = &_Py_STR(utf_8); } else if (encoding == NULL || (strcmp(encoding, "locale") == 0)) { self->encoding = _Py_GetLocaleEncodingObject(); @@ -2267,7 +2267,7 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit) Py_CLEAR(chunks); } if (line == NULL) { - line = Py_NewRef(&_Py_STR(empty)); + line = &_Py_STR(empty); } return line; diff --git a/Modules/_json.c b/Modules/_json.c index 4fcaa07d9cfd81..c7cfe50b52faff 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1277,13 +1277,13 @@ _encoded_const(PyObject *obj) { /* Return the JSON string representation of None, True, False */ if (obj == Py_None) { - return Py_NewRef(&_Py_ID(null)); + return &_Py_ID(null); } else if (obj == Py_True) { - return Py_NewRef(&_Py_ID(true)); + return &_Py_ID(true); } else if (obj == Py_False) { - return Py_NewRef(&_Py_ID(false)); + return &_Py_ID(false); } else { PyErr_SetString(PyExc_ValueError, "not a const"); diff --git a/Modules/_opcode.c b/Modules/_opcode.c index b8d95a048ec2c6..ad0fa736f82767 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -292,7 +292,16 @@ opcode_functions[] = { {NULL, NULL, 0, NULL} }; +int +_opcode_exec(PyObject *m) { + if (PyModule_AddIntMacro(m, ENABLE_SPECIALIZATION) < 0) { + return -1; + } + return 0; +} + static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, _opcode_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Modules/_operator.c b/Modules/_operator.c index b59bfe9ac32171..1f6496d381adac 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1549,10 +1549,77 @@ static PyType_Spec attrgetter_type_spec = { typedef struct { PyObject_HEAD PyObject *name; - PyObject *args; + PyObject *xargs; // reference to arguments passed in constructor PyObject *kwds; + PyObject **vectorcall_args; /* Borrowed references */ + PyObject *vectorcall_kwnames; + vectorcallfunc vectorcall; } methodcallerobject; +static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) +{ + PyObject* args = mc->xargs; + PyObject* kwds = mc->kwds; + + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + assert(nargs > 0); + mc->vectorcall_args = PyMem_Calloc( + nargs + (kwds ? PyDict_Size(kwds) : 0), + sizeof(PyObject*)); + if (!mc->vectorcall_args) { + PyErr_NoMemory(); + return -1; + } + /* The first item of vectorcall_args will be filled with obj later */ + if (nargs > 1) { + memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), + nargs * sizeof(PyObject*)); + } + if (kwds) { + const Py_ssize_t nkwds = PyDict_Size(kwds); + + mc->vectorcall_kwnames = PyTuple_New(nkwds); + if (!mc->vectorcall_kwnames) { + return -1; + } + Py_ssize_t i = 0, ppos = 0; + PyObject* key, * value; + while (PyDict_Next(kwds, &ppos, &key, &value)) { + PyTuple_SET_ITEM(mc->vectorcall_kwnames, i, Py_NewRef(key)); + mc->vectorcall_args[nargs + i] = value; // borrowed reference + ++i; + } + } + else { + mc->vectorcall_kwnames = NULL; + } + return 1; +} + + +static PyObject * +methodcaller_vectorcall( + methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) +{ + if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) + || !_PyArg_NoKwnames("methodcaller", kwnames)) { + return NULL; + } + if (mc->vectorcall_args == NULL) { + if (_methodcaller_initialize_vectorcall(mc) < 0) { + return NULL; + } + } + + assert(mc->vectorcall_args != 0); + mc->vectorcall_args[0] = args[0]; + return PyObject_VectorcallMethod( + mc->name, mc->vectorcall_args, + (PyTuple_GET_SIZE(mc->xargs)) | PY_VECTORCALL_ARGUMENTS_OFFSET, + mc->vectorcall_kwnames); +} + + /* AC 3.5: variable number of arguments, not currently support by AC */ static PyObject * methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -1580,30 +1647,32 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } - name = PyTuple_GET_ITEM(args, 0); Py_INCREF(name); PyUnicode_InternInPlace(&name); mc->name = name; + mc->xargs = Py_XNewRef(args); // allows us to use borrowed references mc->kwds = Py_XNewRef(kwds); + mc->vectorcall_args = 0; - mc->args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); - if (mc->args == NULL) { - Py_DECREF(mc); - return NULL; - } + + mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; PyObject_GC_Track(mc); return (PyObject *)mc; } -static int +static void methodcaller_clear(methodcallerobject *mc) { Py_CLEAR(mc->name); - Py_CLEAR(mc->args); + Py_CLEAR(mc->xargs); Py_CLEAR(mc->kwds); - return 0; + if (mc->vectorcall_args != NULL) { + PyMem_Free(mc->vectorcall_args); + mc->vectorcall_args = 0; + Py_CLEAR(mc->vectorcall_kwnames); + } } static void @@ -1611,7 +1680,7 @@ methodcaller_dealloc(methodcallerobject *mc) { PyTypeObject *tp = Py_TYPE(mc); PyObject_GC_UnTrack(mc); - (void)methodcaller_clear(mc); + methodcaller_clear(mc); tp->tp_free(mc); Py_DECREF(tp); } @@ -1620,7 +1689,7 @@ static int methodcaller_traverse(methodcallerobject *mc, visitproc visit, void *arg) { Py_VISIT(mc->name); - Py_VISIT(mc->args); + Py_VISIT(mc->xargs); Py_VISIT(mc->kwds); Py_VISIT(Py_TYPE(mc)); return 0; @@ -1639,7 +1708,16 @@ methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) method = PyObject_GetAttr(obj, mc->name); if (method == NULL) return NULL; - result = PyObject_Call(method, mc->args, mc->kwds); + + + PyObject *cargs = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); + if (cargs == NULL) { + Py_DECREF(method); + return NULL; + } + + result = PyObject_Call(method, cargs, mc->kwds); + Py_DECREF(cargs); Py_DECREF(method); return result; } @@ -1657,7 +1735,7 @@ methodcaller_repr(methodcallerobject *mc) } numkwdargs = mc->kwds != NULL ? PyDict_GET_SIZE(mc->kwds) : 0; - numposargs = PyTuple_GET_SIZE(mc->args); + numposargs = PyTuple_GET_SIZE(mc->xargs) - 1; numtotalargs = numposargs + numkwdargs; if (numtotalargs == 0) { @@ -1673,7 +1751,7 @@ methodcaller_repr(methodcallerobject *mc) } for (i = 0; i < numposargs; ++i) { - PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->args, i)); + PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->xargs, i+1)); if (onerepr == NULL) goto done; PyTuple_SET_ITEM(argreprs, i, onerepr); @@ -1723,17 +1801,16 @@ methodcaller_repr(methodcallerobject *mc) static PyObject * methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) { - PyObject *newargs; if (!mc->kwds || PyDict_GET_SIZE(mc->kwds) == 0) { Py_ssize_t i; - Py_ssize_t callargcount = PyTuple_GET_SIZE(mc->args); - newargs = PyTuple_New(1 + callargcount); + Py_ssize_t newarg_size = PyTuple_GET_SIZE(mc->xargs); + PyObject *newargs = PyTuple_New(newarg_size); if (newargs == NULL) return NULL; PyTuple_SET_ITEM(newargs, 0, Py_NewRef(mc->name)); - for (i = 0; i < callargcount; ++i) { - PyObject *arg = PyTuple_GET_ITEM(mc->args, i); - PyTuple_SET_ITEM(newargs, i + 1, Py_NewRef(arg)); + for (i = 1; i < newarg_size; ++i) { + PyObject *arg = PyTuple_GET_ITEM(mc->xargs, i); + PyTuple_SET_ITEM(newargs, i, Py_NewRef(arg)); } return Py_BuildValue("ON", Py_TYPE(mc), newargs); } @@ -1751,7 +1828,12 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) constructor = PyObject_VectorcallDict(partial, newargs, 2, mc->kwds); Py_DECREF(partial); - return Py_BuildValue("NO", constructor, mc->args); + PyObject *args = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); + if (!args) { + Py_DECREF(constructor); + return NULL; + } + return Py_BuildValue("NO", constructor, args); } } @@ -1760,6 +1842,12 @@ static PyMethodDef methodcaller_methods[] = { reduce_doc}, {NULL} }; + +static PyMemberDef methodcaller_members[] = { + {"__vectorcalloffset__", Py_T_PYSSIZET, offsetof(methodcallerobject, vectorcall), Py_READONLY}, + {NULL} +}; + PyDoc_STRVAR(methodcaller_doc, "methodcaller(name, /, *args, **kwargs)\n--\n\n\ Return a callable object that calls the given method on its operand.\n\ @@ -1774,6 +1862,7 @@ static PyType_Slot methodcaller_type_slots[] = { {Py_tp_traverse, methodcaller_traverse}, {Py_tp_clear, methodcaller_clear}, {Py_tp_methods, methodcaller_methods}, + {Py_tp_members, methodcaller_members}, {Py_tp_new, methodcaller_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, methodcaller_repr}, @@ -1785,7 +1874,7 @@ static PyType_Spec methodcaller_type_spec = { .basicsize = sizeof(methodcallerobject), .itemsize = 0, .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_IMMUTABLETYPE), .slots = methodcaller_type_slots, }; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d4c0be78935235..c2b04cc513a664 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2029,8 +2029,7 @@ whichmodule(PyObject *global, PyObject *dotted_path) } /* If no module is found, use __main__. */ - module_name = &_Py_ID(__main__); - return Py_NewRef(module_name); + return &_Py_ID(__main__); } /* fast_save_enter() and fast_save_leave() are guards against recursive diff --git a/Modules/_ssl.c b/Modules/_ssl.c index b61d10d9f59fde..c001b875906d44 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -667,6 +667,10 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) errstr = "Some I/O error occurred"; } } else { + if (ERR_GET_LIB(e) == ERR_LIB_SSL && + ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) { + type = state->PySSLCertVerificationErrorObject; + } p = PY_SSL_ERROR_SYSCALL; } break; diff --git a/Modules/_testcapi/vectorcall.c b/Modules/_testcapi/vectorcall.c index 61c6e0f485f47e..2b5110fcba2c91 100644 --- a/Modules/_testcapi/vectorcall.c +++ b/Modules/_testcapi/vectorcall.c @@ -155,10 +155,9 @@ VectorCallClass_vectorcall(PyObject *callable, } /*[clinic input] -module _testcapi class _testcapi.VectorCallClass "PyObject *" "&PyType_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=8423a8e919f2f0df]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=95c63c1a47f9a995]*/ /*[clinic input] _testcapi.VectorCallClass.set_vectorcall diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 246c0a9e160aa9..35a35091bf4511 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -460,6 +460,7 @@ update_refs(PyGC_Head *containers) static int visit_decref(PyObject *op, void *parent) { + OBJECT_STAT_INC(object_visits); _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op)); if (_PyObject_IS_GC(op)) { @@ -498,6 +499,7 @@ subtract_refs(PyGC_Head *containers) static int visit_reachable(PyObject *op, PyGC_Head *reachable) { + OBJECT_STAT_INC(object_visits); if (!_PyObject_IS_GC(op)) { return 0; } @@ -725,6 +727,7 @@ clear_unreachable_mask(PyGC_Head *unreachable) static int visit_move(PyObject *op, PyGC_Head *tolist) { + OBJECT_STAT_INC(object_visits); if (_PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); if (gc_is_collecting(gc)) { @@ -1195,6 +1198,12 @@ gc_collect_main(PyThreadState *tstate, int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, int nofail) { + GC_STAT_ADD(generation, collections, 1); +#ifdef Py_STATS + if (_py_stats) { + _py_stats->object_stats.object_visits = 0; + } +#endif int i; Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ @@ -1351,6 +1360,15 @@ gc_collect_main(PyThreadState *tstate, int generation, stats->collected += m; stats->uncollectable += n; + GC_STAT_ADD(generation, objects_collected, m); +#ifdef Py_STATS + if (_py_stats) { + GC_STAT_ADD(generation, object_visits, + _py_stats->object_stats.object_visits); + _py_stats->object_stats.object_visits = 0; + } +#endif + if (PyDTrace_GC_DONE_ENABLED()) { PyDTrace_GC_DONE(n + m); } diff --git a/Objects/boolobject.c b/Objects/boolobject.c index bbb187cb7121e7..e2e359437f0edf 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -13,8 +13,7 @@ static PyObject * bool_repr(PyObject *self) { - PyObject *res = self == Py_True ? &_Py_ID(True) : &_Py_ID(False); - return Py_NewRef(res); + return self == Py_True ? &_Py_ID(True) : &_Py_ID(False); } /* Function to return a bool from a C long */ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 6b9231a9fa7693..afe9192720c628 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -41,17 +41,12 @@ Py_LOCAL_INLINE(Py_ssize_t) _PyBytesWriter_GetSize(_PyBytesWriter *writer, #define EMPTY (&_Py_SINGLETON(bytes_empty)) -// Return a borrowed reference to the empty bytes string singleton. +// Return a reference to the immortal empty bytes string singleton. static inline PyObject* bytes_get_empty(void) { - return &EMPTY->ob_base.ob_base; -} - - -// Return a strong reference to the empty bytes string singleton. -static inline PyObject* bytes_new_empty(void) -{ - return Py_NewRef(EMPTY); + PyObject *empty = &EMPTY->ob_base.ob_base; + assert(_Py_IsImmortal(empty)); + return empty; } @@ -84,7 +79,7 @@ _PyBytes_FromSize(Py_ssize_t size, int use_calloc) assert(size >= 0); if (size == 0) { - return bytes_new_empty(); + return bytes_get_empty(); } if ((size_t)size > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) { @@ -123,10 +118,11 @@ PyBytes_FromStringAndSize(const char *str, Py_ssize_t size) } if (size == 1 && str != NULL) { op = CHARACTER(*str & 255); - return Py_NewRef(op); + assert(_Py_IsImmortal(op)); + return (PyObject *)op; } if (size == 0) { - return bytes_new_empty(); + return bytes_get_empty(); } op = (PyBytesObject *)_PyBytes_FromSize(size, 0); @@ -154,11 +150,12 @@ PyBytes_FromString(const char *str) } if (size == 0) { - return bytes_new_empty(); + return bytes_get_empty(); } else if (size == 1) { op = CHARACTER(*str & 255); - return Py_NewRef(op); + assert(_Py_IsImmortal(op)); + return (PyObject *)op; } /* Inline PyObject_NewVar */ @@ -3065,7 +3062,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) goto error; } if (newsize == 0) { - *pv = bytes_new_empty(); + *pv = bytes_get_empty(); Py_DECREF(v); return 0; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index cc9ac4b42f479a..17571535048e23 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -879,9 +879,6 @@ frame_dealloc(PyFrameObject *f) /* It is the responsibility of the owning generator/coroutine * to have cleared the generator pointer */ - assert(f->f_frame->owner != FRAME_OWNED_BY_GENERATOR || - _PyFrame_GetGenerator(f->f_frame)->gi_frame_state == FRAME_CLEARED); - if (_PyObject_GC_IS_TRACKED(f)) { _PyObject_GC_UNTRACK(f); } @@ -889,10 +886,14 @@ frame_dealloc(PyFrameObject *f) Py_TRASHCAN_BEGIN(f, frame_dealloc); PyObject *co = NULL; + /* GH-106092: If f->f_frame was on the stack and we reached the maximum + * nesting depth for deallocations, the trashcan may have delayed this + * deallocation until after f->f_frame is freed. Avoid dereferencing + * f->f_frame unless we know it still points to valid memory. */ + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data; + /* Kill all local variables including specials, if we own them */ - if (f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { - assert(f->f_frame == (_PyInterpreterFrame *)f->_f_frame_data); - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data; + if (f->f_frame == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { /* Don't clear code object until the end */ co = frame->f_executable; frame->f_executable = NULL; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 7fffa1c8bbff96..8c0bface3ac710 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -831,8 +831,8 @@ func_clear(PyFunctionObject *op) // However, name and qualname could be str subclasses, so they // could have reference cycles. The solution is to replace them // with a genuinely immutable string. - Py_SETREF(op->func_name, Py_NewRef(&_Py_STR(empty))); - Py_SETREF(op->func_qualname, Py_NewRef(&_Py_STR(empty))); + Py_SETREF(op->func_name, &_Py_STR(empty)); + Py_SETREF(op->func_qualname, &_Py_STR(empty)); return 0; } diff --git a/Objects/genobject.c b/Objects/genobject.c index a630f84fb5a29d..65782be182cd71 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -149,14 +149,16 @@ gen_dealloc(PyGenObject *gen) gen->gi_frame_state = FRAME_CLEARED; frame->previous = NULL; _PyFrame_ClearExceptCode(frame); + _PyErr_ClearExcState(&gen->gi_exc_state); } + assert(gen->gi_exc_state.exc_value == NULL); if (_PyGen_GetCode(gen)->co_flags & CO_COROUTINE) { Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer); } Py_DECREF(_PyGen_GetCode(gen)); Py_CLEAR(gen->gi_name); Py_CLEAR(gen->gi_qualname); - _PyErr_ClearExcState(&gen->gi_exc_state); + PyObject_GC_Del(gen); } @@ -252,10 +254,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, !PyErr_ExceptionMatches(PyExc_StopAsyncIteration)); } - /* generator can't be rerun, so release the frame */ - /* first clean reference cycle through stored exception traceback */ - _PyErr_ClearExcState(&gen->gi_exc_state); - + assert(gen->gi_exc_state.exc_value == NULL); assert(gen->gi_frame_state == FRAME_CLEARED); *presult = result; return result ? PYGEN_RETURN : PYGEN_ERROR; diff --git a/Objects/longobject.c b/Objects/longobject.c index 5d9b413861478a..354cba9d6d800f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -174,7 +174,7 @@ _PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits) { assert(digit_count >= 0); if (digit_count == 0) { - return (PyLongObject *)Py_NewRef(_PyLong_GetZero()); + return (PyLongObject *)_PyLong_GetZero(); } PyLongObject *result = _PyLong_New(digit_count); if (result == NULL) { @@ -2857,8 +2857,7 @@ long_divrem(PyLongObject *a, PyLongObject *b, if (*prem == NULL) { return -1; } - PyObject *zero = _PyLong_GetZero(); - *pdiv = (PyLongObject*)Py_NewRef(zero); + *pdiv = (PyLongObject*)_PyLong_GetZero(); return 0; } if (size_b == 1) { diff --git a/Objects/object.c b/Objects/object.c index 8caa5fd0af3cc9..c4413a00ede80c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -159,11 +159,8 @@ _PyDebug_PrintTotalRefs(void) { Do not call them otherwise, they do not initialize the object! */ #ifdef Py_TRACE_REFS -/* Head of circular doubly-linked list of all objects. These are linked - * together via the _ob_prev and _ob_next members of a PyObject, which - * exist only in a Py_TRACE_REFS build. - */ -static PyObject refchain = {&refchain, &refchain}; + +#define REFCHAIN(interp) &interp->object_state.refchain /* Insert op at the front of the list of all objects. If force is true, * op is added even if _ob_prev and _ob_next are non-NULL already. If @@ -188,10 +185,11 @@ _Py_AddToAllObjects(PyObject *op, int force) } #endif if (force || op->_ob_prev == NULL) { - op->_ob_next = refchain._ob_next; - op->_ob_prev = &refchain; - refchain._ob_next->_ob_prev = op; - refchain._ob_next = op; + PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); + op->_ob_next = refchain->_ob_next; + op->_ob_prev = refchain; + refchain->_ob_next->_ob_prev = op; + refchain->_ob_next = op; } } #endif /* Py_TRACE_REFS */ @@ -2229,7 +2227,8 @@ _Py_ForgetReference(PyObject *op) _PyObject_ASSERT_FAILED_MSG(op, "negative refcnt"); } - if (op == &refchain || + PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); + if (op == refchain || op->_ob_prev->_ob_next != op || op->_ob_next->_ob_prev != op) { _PyObject_ASSERT_FAILED_MSG(op, "invalid object chain"); @@ -2237,12 +2236,12 @@ _Py_ForgetReference(PyObject *op) #ifdef SLOW_UNREF_CHECK PyObject *p; - for (p = refchain._ob_next; p != &refchain; p = p->_ob_next) { + for (p = refchain->_ob_next; p != refchain; p = p->_ob_next) { if (p == op) { break; } } - if (p == &refchain) { + if (p == refchain) { /* Not found */ _PyObject_ASSERT_FAILED_MSG(op, "object not found in the objects list"); @@ -2258,11 +2257,15 @@ _Py_ForgetReference(PyObject *op) * interpreter must be in a healthy state. */ void -_Py_PrintReferences(FILE *fp) +_Py_PrintReferences(PyInterpreterState *interp, FILE *fp) { PyObject *op; + if (interp == NULL) { + interp = _PyInterpreterState_Main(); + } fprintf(fp, "Remaining objects:\n"); - for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) { + PyObject *refchain = REFCHAIN(interp); + for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) { fprintf(fp, "%p [%zd] ", (void *)op, Py_REFCNT(op)); if (PyObject_Print(op, fp, 0) != 0) { PyErr_Clear(); @@ -2274,34 +2277,42 @@ _Py_PrintReferences(FILE *fp) /* Print the addresses of all live objects. Unlike _Py_PrintReferences, this * doesn't make any calls to the Python C API, so is always safe to call. */ +// XXX This function is not safe to use if the interpreter has been +// freed or is in an unhealthy state (e.g. late in finalization). +// The call in Py_FinalizeEx() is okay since the main interpreter +// is statically allocated. void -_Py_PrintReferenceAddresses(FILE *fp) +_Py_PrintReferenceAddresses(PyInterpreterState *interp, FILE *fp) { PyObject *op; + PyObject *refchain = REFCHAIN(interp); fprintf(fp, "Remaining object addresses:\n"); - for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) + for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) fprintf(fp, "%p [%zd] %s\n", (void *)op, Py_REFCNT(op), Py_TYPE(op)->tp_name); } +/* The implementation of sys.getobjects(). */ PyObject * _Py_GetObjects(PyObject *self, PyObject *args) { int i, n; PyObject *t = NULL; PyObject *res, *op; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (!PyArg_ParseTuple(args, "i|O", &n, &t)) return NULL; - op = refchain._ob_next; + PyObject *refchain = REFCHAIN(interp); + op = refchain->_ob_next; res = PyList_New(0); if (res == NULL) return NULL; - for (i = 0; (n == 0 || i < n) && op != &refchain; i++) { + for (i = 0; (n == 0 || i < n) && op != refchain; i++) { while (op == self || op == args || op == res || op == t || (t != NULL && !Py_IS_TYPE(op, (PyTypeObject *) t))) { op = op->_ob_next; - if (op == &refchain) + if (op == refchain) return res; } if (PyList_Append(res, op) < 0) { @@ -2313,6 +2324,8 @@ _Py_GetObjects(PyObject *self, PyObject *args) return res; } +#undef REFCHAIN + #endif diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 1e3d5acc8ae6f8..6e06bef95032cf 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -106,8 +106,8 @@ range_from_array(PyTypeObject *type, PyObject *const *args, Py_ssize_t num_args) if (!stop) { return NULL; } - start = Py_NewRef(_PyLong_GetZero()); - step = Py_NewRef(_PyLong_GetOne()); + start = _PyLong_GetZero(); + step = _PyLong_GetOne(); break; case 0: PyErr_SetString(PyExc_TypeError, diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index dc3aad11ce10e5..8cf654fb6f812d 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -415,7 +415,7 @@ _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, /* Convert step to an integer; raise for zero step. */ if (self->step == Py_None) { - step = Py_NewRef(_PyLong_GetOne()); + step = _PyLong_GetOne(); step_is_negative = 0; } else { @@ -443,7 +443,7 @@ _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, goto error; } else { - lower = Py_NewRef(_PyLong_GetZero()); + lower = _PyLong_GetZero(); upper = Py_NewRef(length); } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index c3ff40fdb14c60..b669a3dd8525eb 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -62,7 +62,7 @@ tuple_alloc(Py_ssize_t size) static inline PyObject * tuple_get_empty(void) { - return Py_NewRef(&_Py_SINGLETON(tuple_empty)); + return (PyObject *)&_Py_SINGLETON(tuple_empty); } PyObject * diff --git a/Objects/typeobject.c b/Objects/typeobject.c index abe33f1562059d..76809494dd8817 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1085,7 +1085,7 @@ type_module(PyTypeObject *type, void *context) PyUnicode_InternInPlace(&mod); } else { - mod = Py_NewRef(&_Py_ID(builtins)); + mod = &_Py_ID(builtins); } } return mod; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index cc979b2ef28d76..c6876d4ca0ef0f 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -211,21 +211,13 @@ static int unicode_is_singleton(PyObject *unicode); #endif -// Return a borrowed reference to the empty string singleton. +// Return a reference to the immortal empty string singleton. static inline PyObject* unicode_get_empty(void) { _Py_DECLARE_STR(empty, ""); return &_Py_STR(empty); } - -// Return a strong reference to the empty string singleton. -static inline PyObject* unicode_new_empty(void) -{ - PyObject *empty = unicode_get_empty(); - return Py_NewRef(empty); -} - /* This dictionary holds all interned unicode strings. Note that references to strings in this dictionary are *not* counted in the string's ob_refcnt. When the interned string reaches a refcnt of 0 the string deallocation @@ -310,7 +302,7 @@ clear_interned_dict(PyInterpreterState *interp) #define _Py_RETURN_UNICODE_EMPTY() \ do { \ - return unicode_new_empty(); \ + return unicode_get_empty(); \ } while (0) static inline void @@ -650,7 +642,6 @@ unicode_result(PyObject *unicode) PyObject *empty = unicode_get_empty(); if (unicode != empty) { Py_DECREF(unicode); - Py_INCREF(empty); } return empty; } @@ -662,7 +653,6 @@ unicode_result(PyObject *unicode) Py_UCS1 ch = data[0]; PyObject *latin1_char = LATIN1(ch); if (unicode != latin1_char) { - Py_INCREF(latin1_char); Py_DECREF(unicode); } return latin1_char; @@ -1199,7 +1189,7 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) { /* Optimization for empty strings */ if (size == 0) { - return unicode_new_empty(); + return unicode_get_empty(); } PyObject *obj; @@ -1669,7 +1659,7 @@ unicode_resize(PyObject **p_unicode, Py_ssize_t length) return 0; if (length == 0) { - PyObject *empty = unicode_new_empty(); + PyObject *empty = unicode_get_empty(); Py_SETREF(*p_unicode, empty); return 0; } @@ -1764,7 +1754,9 @@ unicode_write_cstr(PyObject *unicode, Py_ssize_t index, static PyObject* get_latin1_char(Py_UCS1 ch) { - return Py_NewRef(LATIN1(ch)); + PyObject *o = LATIN1(ch); + assert(_Py_IsImmortal(o)); + return o; } static PyObject* @@ -1891,7 +1883,7 @@ PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) "NULL string with positive size with NULL passed to PyUnicode_FromStringAndSize"); return NULL; } - return unicode_new_empty(); + return unicode_get_empty(); } PyObject * @@ -10261,7 +10253,7 @@ replace(PyObject *self, PyObject *str1, } new_size = slen + n * (len2 - len1); if (new_size == 0) { - u = unicode_new_empty(); + u = unicode_get_empty(); goto done; } if (new_size > (PY_SSIZE_T_MAX / rkind)) { @@ -14505,7 +14497,7 @@ unicode_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, { PyObject *unicode; if (x == NULL) { - unicode = unicode_new_empty(); + unicode = unicode_get_empty(); } else if (encoding == NULL && errors == NULL) { unicode = PyObject_Str(x); @@ -14994,8 +14986,7 @@ unicode_ascii_iter_next(unicodeiterobject *it) Py_UCS1 chr = (Py_UCS1)PyUnicode_READ(PyUnicode_1BYTE_KIND, data, it->it_index); it->it_index++; - PyObject *item = (PyObject*)&_Py_SINGLETON(strings).ascii[chr]; - return Py_NewRef(item); + return (PyObject*)&_Py_SINGLETON(strings).ascii[chr]; } it->it_seq = NULL; Py_DECREF(seq); @@ -15025,7 +15016,7 @@ unicodeiter_reduce(unicodeiterobject *it, PyObject *Py_UNUSED(ignored)) if (it->it_seq != NULL) { return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } else { - PyObject *u = unicode_new_empty(); + PyObject *u = unicode_get_empty(); if (u == NULL) { Py_XDECREF(iter); return NULL; diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 2dd786e5e82e36..99cfff5acc0baf 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -59,9 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - - diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index f159b573ce4727..2a36610527f898 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1393,7 +1393,7 @@ class PartingShots(StaticVisitor): int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; + int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 412de79397477b..8047b1259c5d86 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13073,7 +13073,7 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; + int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/ast.c b/Python/ast.c index 68600ce683b974..74c97f948d15e6 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1029,7 +1029,7 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 int _PyAST_Validate(mod_ty mod) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index ad1e312b084b74..82e7559e5b629a 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1112,7 +1112,7 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_SEQ /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 int _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2b871e8546efb7..90e26d3c86b380 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -741,7 +741,7 @@ dummy_func( tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; } @@ -1348,9 +1348,6 @@ dummy_func( null = NULL; } - op(_SKIP_CACHE, (unused/1 -- )) { - } - op(_GUARD_GLOBALS_VERSION, (version/1 --)) { PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); @@ -1386,13 +1383,13 @@ dummy_func( } macro(LOAD_GLOBAL_MODULE) = - _SKIP_CACHE + // Skip over the counter + unused/1 + // Skip over the counter _GUARD_GLOBALS_VERSION + - _SKIP_CACHE + // Skip over the builtins version + unused/1 + // Skip over the builtins version _LOAD_GLOBAL_MODULE; macro(LOAD_GLOBAL_BUILTIN) = - _SKIP_CACHE + // Skip over the counter + unused/1 + // Skip over the counter _GUARD_GLOBALS_VERSION + _GUARD_BUILTINS_VERSION + _LOAD_GLOBAL_BUILTINS; @@ -1824,7 +1821,7 @@ dummy_func( DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); } - op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- res2 if (oparg & 1), res)) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); res = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(res == NULL, LOAD_ATTR); @@ -1835,10 +1832,11 @@ dummy_func( } macro(LOAD_ATTR_INSTANCE_VALUE) = - _SKIP_CACHE + // Skip over the counter + unused/1 + // Skip over the counter _GUARD_TYPE_VERSION + _CHECK_MANAGED_OBJECT_HAS_VALUES + - _LOAD_ATTR_INSTANCE_VALUE; + _LOAD_ATTR_INSTANCE_VALUE + + unused/5; // Skip over rest of cache inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); @@ -3247,7 +3245,7 @@ dummy_func( total_args++; } DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); PyObject *arg = args[0]; @@ -3274,7 +3272,7 @@ dummy_func( total_args++; } DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); PyObject *cls = args[1]; @@ -3297,7 +3295,7 @@ dummy_func( ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); assert(method != NULL); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyInterpreterState *interp = tstate->interp; DEOPT_IF(method != interp->callable_cache.list_append, CALL); DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); diff --git a/Python/ceval.c b/Python/ceval.c index 17818a0d3c17e6..30f722e45e476b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -624,6 +624,11 @@ extern const struct _PyCode_DEF(8) _Py_InitCleanup; # pragma warning(disable:4102) #endif + +/* _PyEval_EvalFrameDefault() is a *big* function, + * so consume 3 units of C stack */ +#define PY_EVAL_C_STACK_UNITS 2 + PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { @@ -676,6 +681,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int frame->previous = &entry_frame; cframe.current_frame = frame; + tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); if (_Py_EnterRecursiveCallTstate(tstate, "")) { tstate->c_recursion_remaining--; tstate->py_recursion_remaining--; @@ -907,7 +913,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return NULL; } @@ -1259,7 +1265,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (co->co_flags & CO_VARARGS) { PyObject *u = NULL; if (argcount == n) { - u = Py_NewRef(&_Py_SINGLETON(tuple_empty)); + u = (PyObject *)&_Py_SINGLETON(tuple_empty); } else { assert(args != NULL); @@ -1466,6 +1472,7 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) tstate->c_recursion_remaining--; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); _PyFrame_ClearExceptCode(frame); + _PyErr_ClearExcState(&gen->gi_exc_state); tstate->c_recursion_remaining++; frame->previous = NULL; } diff --git a/Python/context.c b/Python/context.c index 9ac51874fb5be5..c94c014219d0e4 100644 --- a/Python/context.c +++ b/Python/context.c @@ -1267,7 +1267,7 @@ PyTypeObject _PyContextTokenMissing_Type = { static PyObject * get_token_missing(void) { - return Py_NewRef(&_Py_SINGLETON(context_token_missing)); + return (PyObject *)&_Py_SINGLETON(context_token_missing); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f3e24bc0479ec2..9363b4955087db 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -47,14 +47,16 @@ } case STORE_FAST: { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; SETLOCAL(oparg, value); STACK_SHRINK(1); break; } case POP_TOP: { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; Py_DECREF(value); STACK_SHRINK(1); break; @@ -69,8 +71,10 @@ } case END_SEND: { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; Py_DECREF(receiver); STACK_SHRINK(1); stack_pointer[-1] = value; @@ -78,8 +82,9 @@ } case UNARY_NEGATIVE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Negative(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -88,8 +93,9 @@ } case UNARY_NOT: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(PyBool_Check(value)); res = Py_IsFalse(value) ? Py_True : Py_False; stack_pointer[-1] = res; @@ -98,8 +104,9 @@ case TO_BOOL: { static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -119,15 +126,17 @@ } case TO_BOOL_BOOL: { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); break; } case TO_BOOL_INT: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { @@ -143,8 +152,9 @@ } case TO_BOOL_LIST: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; @@ -154,8 +164,9 @@ } case TO_BOOL_NONE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); @@ -165,8 +176,9 @@ } case TO_BOOL_STR: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { @@ -183,8 +195,9 @@ } case TO_BOOL_ALWAYS_TRUE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; uint32_t version = (uint32_t)operand; // This one is a bit weird, because we expect *some* failures: assert(version); @@ -197,8 +210,9 @@ } case UNARY_INVERT: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Invert(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -207,17 +221,21 @@ } case _GUARD_BOTH_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); break; } case _BINARY_OP_MULTIPLY_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -229,9 +247,11 @@ } case _BINARY_OP_ADD_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -243,9 +263,11 @@ } case _BINARY_OP_SUBTRACT_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -257,17 +279,21 @@ } case _GUARD_BOTH_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); break; } case _BINARY_OP_MULTIPLY_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval * @@ -279,9 +305,11 @@ } case _BINARY_OP_ADD_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval + @@ -293,9 +321,11 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval - @@ -307,17 +337,21 @@ } case _GUARD_BOTH_UNICODE: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); break; } case _BINARY_OP_ADD_UNICODE: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -330,9 +364,11 @@ case BINARY_SUBSCR: { static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; PyObject *res; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -353,10 +389,13 @@ } case BINARY_SLICE: { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; + PyObject *stop; + PyObject *start; + PyObject *container; PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -375,10 +414,14 @@ } case STORE_SLICE: { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; - PyObject *v = stack_pointer[-4]; + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -396,9 +439,11 @@ } case BINARY_SUBSCR_LIST_INT: { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; + PyObject *sub; + PyObject *list; PyObject *res; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -418,9 +463,11 @@ } case BINARY_SUBSCR_TUPLE_INT: { - PyObject *sub = stack_pointer[-1]; - PyObject *tuple = stack_pointer[-2]; + PyObject *sub; + PyObject *tuple; PyObject *res; + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -440,9 +487,11 @@ } case BINARY_SUBSCR_DICT: { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; + PyObject *sub; + PyObject *dict; PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -463,16 +512,20 @@ } case LIST_APPEND: { - PyObject *v = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; STACK_SHRINK(1); break; } case SET_ADD: { - PyObject *v = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; @@ -482,9 +535,12 @@ case STORE_SUBSCR: { static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; - PyObject *v = stack_pointer[-3]; + PyObject *sub; + PyObject *container; + PyObject *v; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -506,9 +562,12 @@ } case STORE_SUBSCR_LIST_INT: { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -530,9 +589,12 @@ } case STORE_SUBSCR_DICT: { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); @@ -543,8 +605,10 @@ } case DELETE_SUBSCR: { - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; /* del container[sub] */ int err = PyObject_DelItem(container, sub); Py_DECREF(container); @@ -555,8 +619,9 @@ } case CALL_INTRINSIC_1: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); @@ -566,9 +631,11 @@ } case CALL_INTRINSIC_2: { - PyObject *value1 = stack_pointer[-1]; - PyObject *value2 = stack_pointer[-2]; + PyObject *value1; + PyObject *value2; PyObject *res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); @@ -580,8 +647,9 @@ } case GET_AITER: { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *iter; + obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -617,8 +685,9 @@ } case GET_ANEXT: { - PyObject *aiter = stack_pointer[-1]; + PyObject *aiter; PyObject *awaitable; + aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -667,8 +736,9 @@ } case GET_AWAITABLE: { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { @@ -697,7 +767,8 @@ } case POP_EXCEPT: { - PyObject *exc_value = stack_pointer[-1]; + PyObject *exc_value; + exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); STACK_SHRINK(1); @@ -726,7 +797,8 @@ } case STORE_NAME: { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -768,7 +840,8 @@ case UNPACK_SEQUENCE: { static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -789,8 +862,10 @@ } case UNPACK_SEQUENCE_TWO_TUPLE: { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); @@ -804,8 +879,10 @@ } case UNPACK_SEQUENCE_TUPLE: { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -820,8 +897,10 @@ } case UNPACK_SEQUENCE_LIST: { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -836,7 +915,8 @@ } case UNPACK_EX: { - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); @@ -848,8 +928,10 @@ case STORE_ATTR: { static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; - PyObject *v = stack_pointer[-2]; + PyObject *owner; + PyObject *v; + owner = stack_pointer[-1]; + v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -871,7 +953,8 @@ } case DELETE_ATTR: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); @@ -881,7 +964,8 @@ } case STORE_GLOBAL: { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); @@ -920,8 +1004,9 @@ } case _LOAD_FROM_DICT_OR_GLOBALS: { - PyObject *mod_or_class_dict = stack_pointer[-1]; + PyObject *mod_or_class_dict; PyObject *v; + mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { Py_DECREF(mod_or_class_dict); @@ -1004,12 +1089,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = v; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } - break; - } - - case _SKIP_CACHE: { break; } @@ -1044,8 +1125,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } break; } @@ -1062,8 +1143,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } break; } @@ -1089,8 +1170,9 @@ } case LOAD_FROM_DICT_OR_DEREF: { - PyObject *class_dict = stack_pointer[-1]; + PyObject *class_dict; PyObject *value; + class_dict = stack_pointer[-1]; PyObject *name; assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); @@ -1128,7 +1210,8 @@ } case STORE_DEREF: { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); @@ -1152,8 +1235,9 @@ } case BUILD_STRING: { - PyObject **pieces = (stack_pointer - oparg); + PyObject **pieces; PyObject *str; + pieces = stack_pointer - oparg; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); @@ -1166,8 +1250,9 @@ } case BUILD_TUPLE: { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *tup; + values = stack_pointer - oparg; tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1177,8 +1262,9 @@ } case BUILD_LIST: { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *list; + values = stack_pointer - oparg; list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1188,8 +1274,10 @@ } case LIST_EXTEND: { - PyObject *iterable = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1210,8 +1298,10 @@ } case SET_UPDATE: { - PyObject *iterable = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; @@ -1220,8 +1310,9 @@ } case BUILD_SET: { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *set; + values = stack_pointer - oparg; set = PySet_New(NULL); if (set == NULL) goto error; @@ -1243,8 +1334,9 @@ } case BUILD_MAP: { - PyObject **values = (stack_pointer - oparg*2); + PyObject **values; PyObject *map; + values = stack_pointer - oparg*2; map = _PyDict_FromItems( values, 2, values+1, 2, @@ -1304,9 +1396,11 @@ } case BUILD_CONST_KEY_MAP: { - PyObject *keys = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg)); + PyObject *keys; + PyObject **values; PyObject *map; + keys = stack_pointer[-1]; + values = stack_pointer - 1 - oparg; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1327,7 +1421,8 @@ } case DICT_UPDATE: { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -1344,7 +1439,8 @@ } case DICT_MERGE: { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { @@ -1358,8 +1454,10 @@ } case MAP_ADD: { - PyObject *value = stack_pointer[-1]; - PyObject *key = stack_pointer[-2]; + PyObject *value; + PyObject *key; + value = stack_pointer[-1]; + key = stack_pointer[-2]; PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ @@ -1370,11 +1468,14 @@ } case LOAD_SUPER_ATTR_ATTR: { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2 = NULL; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -1387,17 +1488,20 @@ if (res == NULL) goto pop_3_error; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } break; } case LOAD_SUPER_ATTR_METHOD: { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -1421,16 +1525,17 @@ res2 = NULL; } STACK_SHRINK(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; break; } case LOAD_ATTR: { static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1477,13 +1582,14 @@ if (res == NULL) goto pop_1_error; } STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } break; } case _GUARD_TYPE_VERSION: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -1492,7 +1598,8 @@ } case _CHECK_MANAGED_OBJECT_HAS_VALUES: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); @@ -1501,9 +1608,10 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint16_t index = (uint16_t)operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); res = _PyDictOrValues_GetValues(dorv)->values[index]; @@ -1513,16 +1621,18 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } break; } case COMPARE_OP: { static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1550,9 +1660,11 @@ } case COMPARE_OP_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -1570,9 +1682,11 @@ } case COMPARE_OP_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -1594,9 +1708,11 @@ } case COMPARE_OP_STR: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -1615,9 +1731,11 @@ } case IS_OP: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); @@ -1628,9 +1746,11 @@ } case CONTAINS_OP: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); @@ -1642,10 +1762,12 @@ } case CHECK_EG_MATCH: { - PyObject *match_type = stack_pointer[-1]; - PyObject *exc_value = stack_pointer[-2]; + PyObject *match_type; + PyObject *exc_value; PyObject *rest; PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); @@ -1666,15 +1788,17 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - stack_pointer[-1] = match; stack_pointer[-2] = rest; + stack_pointer[-1] = match; break; } case CHECK_EXC_MATCH: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); @@ -1689,8 +1813,9 @@ } case IS_NONE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *b; + value = stack_pointer[-1]; if (Py_IsNone(value)) { b = Py_True; } @@ -1703,8 +1828,9 @@ } case GET_LEN: { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *len_o; + obj = stack_pointer[-1]; // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; @@ -1716,10 +1842,13 @@ } case MATCH_CLASS: { - PyObject *names = stack_pointer[-1]; - PyObject *type = stack_pointer[-2]; - PyObject *subject = stack_pointer[-3]; + PyObject *names; + PyObject *type; + PyObject *subject; PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); @@ -1740,8 +1869,9 @@ } case MATCH_MAPPING: { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -1750,8 +1880,9 @@ } case MATCH_SEQUENCE: { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -1760,9 +1891,11 @@ } case MATCH_KEYS: { - PyObject *keys = stack_pointer[-1]; - PyObject *subject = stack_pointer[-2]; + PyObject *keys; + PyObject *subject; PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; @@ -1772,8 +1905,9 @@ } case GET_ITER: { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); @@ -1783,8 +1917,9 @@ } case GET_YIELD_FROM_ITER: { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -1814,14 +1949,16 @@ } case _ITER_CHECK_LIST: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); break; } case _IS_ITER_EXHAUSTED_LIST: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *exhausted; + iter = stack_pointer[-1]; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; @@ -1842,8 +1979,9 @@ } case _ITER_NEXT_LIST: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; @@ -1856,14 +1994,16 @@ } case _ITER_CHECK_TUPLE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); break; } case _IS_ITER_EXHAUSTED_TUPLE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *exhausted; + iter = stack_pointer[-1]; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; @@ -1884,8 +2024,9 @@ } case _ITER_NEXT_TUPLE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; @@ -1898,15 +2039,17 @@ } case _ITER_CHECK_RANGE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); break; } case _IS_ITER_EXHAUSTED_RANGE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *exhausted; + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); exhausted = r->len <= 0 ? Py_True : Py_False; @@ -1916,8 +2059,9 @@ } case _ITER_NEXT_RANGE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); @@ -1932,10 +2076,13 @@ } case WITH_EXCEPT_START: { - PyObject *val = stack_pointer[-1]; - PyObject *lasti = stack_pointer[-3]; - PyObject *exit_func = stack_pointer[-4]; + PyObject *val; + PyObject *lasti; + PyObject *exit_func; PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -1967,8 +2114,9 @@ } case PUSH_EXC_INFO: { - PyObject *new_exc = stack_pointer[-1]; + PyObject *new_exc; PyObject *prev_exc; + new_exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -1979,16 +2127,19 @@ assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); STACK_GROW(1); - stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; + stack_pointer[-1] = new_exc; break; } case CALL_NO_KW_TYPE_1: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2005,10 +2156,13 @@ } case CALL_NO_KW_STR_1: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2027,10 +2181,13 @@ } case CALL_NO_KW_TUPLE_1: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2049,7 +2206,8 @@ } case EXIT_INIT_CHECK: { - PyObject *should_be_none = stack_pointer[-1]; + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -2062,10 +2220,13 @@ } case CALL_NO_KW_BUILTIN_O: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -2101,10 +2262,13 @@ } case CALL_NO_KW_BUILTIN_FAST: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -2144,10 +2308,13 @@ } case CALL_NO_KW_LEN: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* len(o) */ int is_meth = method != NULL; @@ -2158,7 +2325,7 @@ total_args++; } DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); PyObject *arg = args[0]; @@ -2179,10 +2346,13 @@ } case CALL_NO_KW_ISINSTANCE: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -2193,7 +2363,7 @@ total_args++; } DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); PyObject *cls = args[1]; @@ -2216,9 +2386,11 @@ } case CALL_NO_KW_METHOD_DESCRIPTOR_O: { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -2257,9 +2429,11 @@ } case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -2296,9 +2470,11 @@ } case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -2334,8 +2510,9 @@ } case MAKE_FUNCTION: { - PyObject *codeobj = stack_pointer[-1]; + PyObject *codeobj; PyObject *func; + codeobj = stack_pointer[-1]; PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -2352,8 +2529,10 @@ } case SET_FUNCTION_ATTRIBUTE: { - PyObject *func = stack_pointer[-1]; - PyObject *attr = stack_pointer[-2]; + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -2384,10 +2563,13 @@ } case BUILD_SLICE: { - PyObject *step = (oparg == 3) ? stack_pointer[-(((oparg == 3) ? 1 : 0))] : NULL; - PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; - PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; + PyObject *step = NULL; + PyObject *stop; + PyObject *start; PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } + stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; + start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); @@ -2400,8 +2582,9 @@ } case CONVERT_VALUE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *result; + value = stack_pointer[-1]; convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; @@ -2413,8 +2596,9 @@ } case FORMAT_SIMPLE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -2430,9 +2614,11 @@ } case FORMAT_WITH_SPEC: { - PyObject *fmt_spec = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *fmt_spec; + PyObject *value; PyObject *res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); @@ -2443,8 +2629,9 @@ } case COPY: { - PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; + PyObject *bottom; PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = Py_NewRef(bottom); STACK_GROW(1); @@ -2454,9 +2641,11 @@ case BINARY_OP: { static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *rhs = stack_pointer[-1]; - PyObject *lhs = stack_pointer[-2]; + PyObject *rhs; + PyObject *lhs; PyObject *res; + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2480,16 +2669,19 @@ } case SWAP: { - PyObject *top = stack_pointer[-1]; - PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; stack_pointer[-1] = bottom; - stack_pointer[-(2 + (oparg-2))] = top; break; } case _POP_JUMP_IF_FALSE: { - PyObject *flag = stack_pointer[-1]; + PyObject *flag; + flag = stack_pointer[-1]; if (Py_IsFalse(flag)) { pc = oparg; } @@ -2498,7 +2690,8 @@ } case _POP_JUMP_IF_TRUE: { - PyObject *flag = stack_pointer[-1]; + PyObject *flag; + flag = stack_pointer[-1]; if (Py_IsTrue(flag)) { pc = oparg; } diff --git a/Python/fileutils.c b/Python/fileutils.c index f262c3e095c9ba..19b23f6bd18b30 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -105,7 +105,7 @@ _Py_device_encoding(int fd) #else if (_PyRuntime.preconfig.utf8_mode) { _Py_DECLARE_STR(utf_8, "utf-8"); - return Py_NewRef(&_Py_STR(utf_8)); + return &_Py_STR(utf_8); } return _Py_GetLocaleEncodingObject(); #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6ac622b11acac9..7250240ac2396c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -91,8 +91,8 @@ Py_INCREF(value1); Py_INCREF(value2); STACK_GROW(2); - stack_pointer[-1] = value2; stack_pointer[-2] = value1; + stack_pointer[-1] = value2; DISPATCH(); } @@ -106,15 +106,17 @@ } TARGET(STORE_FAST) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; SETLOCAL(oparg, value); STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_FAST_LOAD_FAST) { - PyObject *value1 = stack_pointer[-1]; + PyObject *value1; PyObject *value2; + value1 = stack_pointer[-1]; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); @@ -125,8 +127,10 @@ } TARGET(STORE_FAST_STORE_FAST) { - PyObject *value1 = stack_pointer[-1]; - PyObject *value2 = stack_pointer[-2]; + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); @@ -136,7 +140,8 @@ } TARGET(POP_TOP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -151,14 +156,15 @@ } TARGET(END_FOR) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *value; + // POP_TOP + value = stack_pointer[-1]; { - PyObject *value = _tmp_1; Py_DECREF(value); } + // POP_TOP + value = stack_pointer[-2]; { - PyObject *value = _tmp_2; Py_DECREF(value); } STACK_SHRINK(2); @@ -166,8 +172,10 @@ } TARGET(INSTRUMENTED_END_FOR) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -184,8 +192,10 @@ } TARGET(END_SEND) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; Py_DECREF(receiver); STACK_SHRINK(1); stack_pointer[-1] = value; @@ -193,8 +203,10 @@ } TARGET(INSTRUMENTED_END_SEND) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -209,8 +221,9 @@ } TARGET(UNARY_NEGATIVE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Negative(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -219,8 +232,9 @@ } TARGET(UNARY_NOT) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(PyBool_Check(value)); res = Py_IsFalse(value) ? Py_True : Py_False; stack_pointer[-1] = res; @@ -230,8 +244,9 @@ TARGET(TO_BOOL) { PREDICTED(TO_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -252,7 +267,8 @@ } TARGET(TO_BOOL_BOOL) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); next_instr += 3; @@ -260,8 +276,9 @@ } TARGET(TO_BOOL_INT) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { @@ -278,8 +295,9 @@ } TARGET(TO_BOOL_LIST) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; @@ -290,8 +308,9 @@ } TARGET(TO_BOOL_NONE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); @@ -302,8 +321,9 @@ } TARGET(TO_BOOL_STR) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { @@ -321,8 +341,9 @@ } TARGET(TO_BOOL_ALWAYS_TRUE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; uint32_t version = read_u32(&next_instr[1].cache); // This one is a bit weird, because we expect *some* failures: assert(version); @@ -336,8 +357,9 @@ } TARGET(UNARY_INVERT) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Invert(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -346,215 +368,192 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_MULTIPLY_INT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_INT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_ADD_INT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_INT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_SUBTRACT_INT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_FLOAT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_MULTIPLY_FLOAT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_FLOAT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_ADD_FLOAT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_FLOAT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_SUBTRACT_FLOAT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_UNICODE) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_ADD_UNICODE { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_INPLACE_ADD_UNICODE { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; assert(true_next.op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(true_next.op.arg); @@ -586,9 +585,11 @@ TARGET(BINARY_SUBSCR) { PREDICTED(BINARY_SUBSCR); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; PyObject *res; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -610,10 +611,13 @@ } TARGET(BINARY_SLICE) { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; + PyObject *stop; + PyObject *start; + PyObject *container; PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -632,10 +636,14 @@ } TARGET(STORE_SLICE) { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; - PyObject *v = stack_pointer[-4]; + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -653,9 +661,11 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; + PyObject *sub; + PyObject *list; PyObject *res; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -676,9 +686,11 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *tuple = stack_pointer[-2]; + PyObject *sub; + PyObject *tuple; PyObject *res; + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -699,9 +711,11 @@ } TARGET(BINARY_SUBSCR_DICT) { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; + PyObject *sub; + PyObject *dict; PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -723,8 +737,10 @@ } TARGET(BINARY_SUBSCR_GETITEM) { - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); @@ -747,19 +763,24 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_SHRINK(1); } TARGET(LIST_APPEND) { - PyObject *v = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; STACK_SHRINK(1); DISPATCH(); } TARGET(SET_ADD) { - PyObject *v = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; @@ -770,9 +791,12 @@ TARGET(STORE_SUBSCR) { PREDICTED(STORE_SUBSCR); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; - PyObject *v = stack_pointer[-3]; + PyObject *sub; + PyObject *container; + PyObject *v; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -795,9 +819,12 @@ } TARGET(STORE_SUBSCR_LIST_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -820,9 +847,12 @@ } TARGET(STORE_SUBSCR_DICT) { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); @@ -834,8 +864,10 @@ } TARGET(DELETE_SUBSCR) { - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; /* del container[sub] */ int err = PyObject_DelItem(container, sub); Py_DECREF(container); @@ -846,8 +878,9 @@ } TARGET(CALL_INTRINSIC_1) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); @@ -857,9 +890,11 @@ } TARGET(CALL_INTRINSIC_2) { - PyObject *value1 = stack_pointer[-1]; - PyObject *value2 = stack_pointer[-2]; + PyObject *value1; + PyObject *value2; PyObject *res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); @@ -871,7 +906,8 @@ } TARGET(RAISE_VARARGS) { - PyObject **args = (stack_pointer - oparg); + PyObject **args; + args = stack_pointer - oparg; PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -893,22 +929,26 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } + STACK_SHRINK(oparg); } TARGET(INTERPRETER_EXIT) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; + STACK_SHRINK(1); } TARGET(RETURN_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -921,10 +961,12 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + STACK_SHRINK(1); } TARGET(INSTRUMENTED_RETURN_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -941,6 +983,7 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + STACK_SHRINK(1); } TARGET(RETURN_CONST) { @@ -980,8 +1023,9 @@ } TARGET(GET_AITER) { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *iter; + obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -1017,8 +1061,9 @@ } TARGET(GET_ANEXT) { - PyObject *aiter = stack_pointer[-1]; + PyObject *aiter; PyObject *awaitable; + aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -1067,8 +1112,9 @@ } TARGET(GET_AWAITABLE) { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { @@ -1099,9 +1145,11 @@ TARGET(SEND) { PREDICTED(SEND); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *v = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *v; + PyObject *receiver; PyObject *retval; + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1154,8 +1202,10 @@ } TARGET(SEND_GEN) { - PyObject *v = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *v; + PyObject *receiver; + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, SEND); PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && @@ -1174,7 +1224,8 @@ } TARGET(INSTRUMENTED_YIELD_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; assert(frame != &entry_frame); assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ PyGenObject *gen = _PyFrame_GetGenerator(frame); @@ -1195,7 +1246,8 @@ } TARGET(YIELD_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1215,7 +1267,8 @@ } TARGET(POP_EXCEPT) { - PyObject *exc_value = stack_pointer[-1]; + PyObject *exc_value; + exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); STACK_SHRINK(1); @@ -1223,8 +1276,10 @@ } TARGET(RERAISE) { - PyObject *exc = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg)); + PyObject *exc; + PyObject **values; + exc = stack_pointer[-1]; + values = stack_pointer - 1 - oparg; assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1243,11 +1298,14 @@ _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, next_instr-1); goto exception_unwind; + STACK_SHRINK(1); } TARGET(END_ASYNC_FOR) { - PyObject *exc = stack_pointer[-1]; - PyObject *awaitable = stack_pointer[-2]; + PyObject *exc; + PyObject *awaitable; + exc = stack_pointer[-1]; + awaitable = stack_pointer[-2]; assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { Py_DECREF(awaitable); @@ -1264,11 +1322,14 @@ } TARGET(CLEANUP_THROW) { - PyObject *exc_value = stack_pointer[-1]; - PyObject *last_sent_val = stack_pointer[-2]; - PyObject *sub_iter = stack_pointer[-3]; + PyObject *exc_value; + PyObject *last_sent_val; + PyObject *sub_iter; PyObject *none; PyObject *value; + exc_value = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { @@ -1284,8 +1345,8 @@ goto exception_unwind; } STACK_SHRINK(1); - stack_pointer[-1] = value; stack_pointer[-2] = none; + stack_pointer[-1] = value; DISPATCH(); } @@ -1311,7 +1372,8 @@ } TARGET(STORE_NAME) { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -1354,7 +1416,8 @@ TARGET(UNPACK_SEQUENCE) { PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1376,8 +1439,10 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); @@ -1392,8 +1457,10 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1409,8 +1476,10 @@ } TARGET(UNPACK_SEQUENCE_LIST) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1426,7 +1495,8 @@ } TARGET(UNPACK_EX) { - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); @@ -1439,8 +1509,10 @@ TARGET(STORE_ATTR) { PREDICTED(STORE_ATTR); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; - PyObject *v = stack_pointer[-2]; + PyObject *owner; + PyObject *v; + owner = stack_pointer[-1]; + v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1463,7 +1535,8 @@ } TARGET(DELETE_ATTR) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); @@ -1473,7 +1546,8 @@ } TARGET(STORE_GLOBAL) { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); @@ -1498,27 +1572,25 @@ } TARGET(LOAD_LOCALS) { - PyObject *_tmp_1; - { - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - Py_INCREF(locals); - _tmp_1 = locals; + PyObject *locals; + locals = LOCALS(); + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; } + Py_INCREF(locals); STACK_GROW(1); - stack_pointer[-1] = _tmp_1; + stack_pointer[-1] = locals; DISPATCH(); } TARGET(LOAD_NAME) { - PyObject *_tmp_1; + PyObject *locals; + PyObject *mod_or_class_dict; + PyObject *v; + // _LOAD_LOCALS { - PyObject *locals; locals = LOCALS(); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1526,11 +1598,10 @@ if (true) goto error; } Py_INCREF(locals); - _tmp_1 = locals; } + // _LOAD_FROM_DICT_OR_GLOBALS + mod_or_class_dict = locals; { - PyObject *mod_or_class_dict = _tmp_1; - PyObject *v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { Py_DECREF(mod_or_class_dict); @@ -1557,47 +1628,43 @@ } } } - _tmp_1 = v; } STACK_GROW(1); - stack_pointer[-1] = _tmp_1; + stack_pointer[-1] = v; DISPATCH(); } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - PyObject *_tmp_1 = stack_pointer[-1]; - { - PyObject *mod_or_class_dict = _tmp_1; - PyObject *v; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - Py_DECREF(mod_or_class_dict); + PyObject *mod_or_class_dict; + PyObject *v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + Py_DECREF(mod_or_class_dict); + goto error; + } + Py_DECREF(mod_or_class_dict); + if (v == NULL) { + v = PyDict_GetItemWithError(GLOBALS(), name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { goto error; } - Py_DECREF(mod_or_class_dict); - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + else { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { goto error; } - else { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - goto error; - } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; } } - _tmp_1 = v; } - stack_pointer[-1] = _tmp_1; + stack_pointer[-1] = v; DISPATCH(); } @@ -1654,17 +1721,16 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = v; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_MODULE) { - PyObject *_tmp_1; - PyObject *_tmp_2; - { - } + PyObject *null = NULL; + PyObject *res; + // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1672,11 +1738,8 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + // _LOAD_GLOBAL_MODULE { - } - { - PyObject *null = NULL; - PyObject *res; uint16_t index = read_u16(&next_instr[3].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); @@ -1685,22 +1748,19 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - if (oparg & 1) { _tmp_2 = null; } - _tmp_1 = res; } - next_instr += 4; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg & 1) { stack_pointer[-2] = _tmp_2; } + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_BUILTIN) { - PyObject *_tmp_1; - PyObject *_tmp_2; - { - } + PyObject *null = NULL; + PyObject *res; + // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1708,6 +1768,7 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + // _GUARD_BUILTINS_VERSION { uint16_t version = read_u16(&next_instr[2].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); @@ -1715,9 +1776,8 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + // _LOAD_GLOBAL_BUILTINS { - PyObject *null = NULL; - PyObject *res; uint16_t index = read_u16(&next_instr[3].cache); PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); @@ -1726,14 +1786,12 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - if (oparg & 1) { _tmp_2 = null; } - _tmp_1 = res; } - next_instr += 4; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg & 1) { stack_pointer[-2] = _tmp_2; } + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } @@ -1771,8 +1829,9 @@ } TARGET(LOAD_FROM_DICT_OR_DEREF) { - PyObject *class_dict = stack_pointer[-1]; + PyObject *class_dict; PyObject *value; + class_dict = stack_pointer[-1]; PyObject *name; assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); @@ -1810,7 +1869,8 @@ } TARGET(STORE_DEREF) { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); @@ -1834,8 +1894,9 @@ } TARGET(BUILD_STRING) { - PyObject **pieces = (stack_pointer - oparg); + PyObject **pieces; PyObject *str; + pieces = stack_pointer - oparg; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); @@ -1848,8 +1909,9 @@ } TARGET(BUILD_TUPLE) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *tup; + values = stack_pointer - oparg; tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1859,8 +1921,9 @@ } TARGET(BUILD_LIST) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *list; + values = stack_pointer - oparg; list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1870,8 +1933,10 @@ } TARGET(LIST_EXTEND) { - PyObject *iterable = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1892,8 +1957,10 @@ } TARGET(SET_UPDATE) { - PyObject *iterable = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; @@ -1902,8 +1969,9 @@ } TARGET(BUILD_SET) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *set; + values = stack_pointer - oparg; set = PySet_New(NULL); if (set == NULL) goto error; @@ -1925,8 +1993,9 @@ } TARGET(BUILD_MAP) { - PyObject **values = (stack_pointer - oparg*2); + PyObject **values; PyObject *map; + values = stack_pointer - oparg*2; map = _PyDict_FromItems( values, 2, values+1, 2, @@ -1986,9 +2055,11 @@ } TARGET(BUILD_CONST_KEY_MAP) { - PyObject *keys = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg)); + PyObject *keys; + PyObject **values; PyObject *map; + keys = stack_pointer[-1]; + values = stack_pointer - 1 - oparg; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2009,7 +2080,8 @@ } TARGET(DICT_UPDATE) { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2026,7 +2098,8 @@ } TARGET(DICT_MERGE) { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { @@ -2040,8 +2113,10 @@ } TARGET(MAP_ADD) { - PyObject *value = stack_pointer[-1]; - PyObject *key = stack_pointer[-2]; + PyObject *value; + PyObject *key; + value = stack_pointer[-1]; + key = stack_pointer[-2]; PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ @@ -2057,16 +2132,21 @@ // don't want to specialize instrumented instructions INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + STACK_SHRINK(2); + STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(LOAD_SUPER_ATTR) { PREDICTED(LOAD_SUPER_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2 = NULL; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION @@ -2117,18 +2197,21 @@ if (res == NULL) goto pop_3_error; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 1; DISPATCH(); } TARGET(LOAD_SUPER_ATTR_ATTR) { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2 = NULL; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -2141,18 +2224,21 @@ if (res == NULL) goto pop_3_error; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 1; DISPATCH(); } TARGET(LOAD_SUPER_ATTR_METHOD) { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -2176,8 +2262,8 @@ res2 = NULL; } STACK_SHRINK(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 1; DISPATCH(); } @@ -2185,9 +2271,10 @@ TARGET(LOAD_ATTR) { PREDICTED(LOAD_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2234,37 +2321,33 @@ if (res == NULL) goto pop_1_error; } STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; - { - } + PyObject *owner; + PyObject *res2 = NULL; + PyObject *res; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - PyObject *owner = _tmp_2; uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - _tmp_2 = owner; } + // _CHECK_MANAGED_OBJECT_HAS_VALUES { - PyObject *owner = _tmp_2; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - _tmp_2 = owner; } + // _LOAD_ATTR_INSTANCE_VALUE { - PyObject *owner = _tmp_2; - PyObject *res2 = NULL; - PyObject *res; uint16_t index = read_u16(&next_instr[3].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); res = _PyDictOrValues_GetValues(dorv)->values[index]; @@ -2273,20 +2356,19 @@ Py_INCREF(res); res2 = NULL; Py_DECREF(owner); - if (oparg & 1) { _tmp_2 = res2; } - _tmp_1 = res; } - next_instr += 9; STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg & 1) { stack_pointer[-2] = _tmp_2; } + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } + stack_pointer[-1] = res; + next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); @@ -2303,16 +2385,17 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2343,16 +2426,17 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2366,16 +2450,17 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { - PyObject *cls = stack_pointer[-1]; + PyObject *cls; PyObject *res2 = NULL; PyObject *res; + cls = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); @@ -2391,14 +2476,15 @@ Py_INCREF(res); Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); @@ -2425,10 +2511,12 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); @@ -2457,11 +2545,14 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(STORE_ATTR_INSTANCE_VALUE) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2487,8 +2578,10 @@ } TARGET(STORE_ATTR_WITH_HINT) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2535,8 +2628,10 @@ } TARGET(STORE_ATTR_SLOT) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2556,9 +2651,11 @@ TARGET(COMPARE_OP) { PREDICTED(COMPARE_OP); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2587,9 +2684,11 @@ } TARGET(COMPARE_OP_FLOAT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2608,9 +2707,11 @@ } TARGET(COMPARE_OP_INT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2633,9 +2734,11 @@ } TARGET(COMPARE_OP_STR) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2655,9 +2758,11 @@ } TARGET(IS_OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); @@ -2668,9 +2773,11 @@ } TARGET(CONTAINS_OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); @@ -2682,10 +2789,12 @@ } TARGET(CHECK_EG_MATCH) { - PyObject *match_type = stack_pointer[-1]; - PyObject *exc_value = stack_pointer[-2]; + PyObject *match_type; + PyObject *exc_value; PyObject *rest; PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); @@ -2706,15 +2815,17 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - stack_pointer[-1] = match; stack_pointer[-2] = rest; + stack_pointer[-1] = match; DISPATCH(); } TARGET(CHECK_EXC_MATCH) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); @@ -2729,9 +2840,11 @@ } TARGET(IMPORT_NAME) { - PyObject *fromlist = stack_pointer[-1]; - PyObject *level = stack_pointer[-2]; + PyObject *fromlist; + PyObject *level; PyObject *res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_name(tstate, frame, name, fromlist, level); Py_DECREF(level); @@ -2743,8 +2856,9 @@ } TARGET(IMPORT_FROM) { - PyObject *from = stack_pointer[-1]; + PyObject *from; PyObject *res; + from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; @@ -2805,7 +2919,8 @@ } TARGET(POP_JUMP_IF_FALSE) { - PyObject *cond = stack_pointer[-1]; + PyObject *cond; + cond = stack_pointer[-1]; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsFalse(cond)); STACK_SHRINK(1); @@ -2813,7 +2928,8 @@ } TARGET(POP_JUMP_IF_TRUE) { - PyObject *cond = stack_pointer[-1]; + PyObject *cond; + cond = stack_pointer[-1]; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsTrue(cond)); STACK_SHRINK(1); @@ -2821,10 +2937,12 @@ } TARGET(POP_JUMP_IF_NONE) { - PyObject *_tmp_1 = stack_pointer[-1]; + PyObject *value; + PyObject *b; + PyObject *cond; + // IS_NONE + value = stack_pointer[-1]; { - PyObject *value = _tmp_1; - PyObject *b; if (Py_IsNone(value)) { b = Py_True; } @@ -2832,10 +2950,10 @@ b = Py_False; Py_DECREF(value); } - _tmp_1 = b; } + // POP_JUMP_IF_TRUE + cond = b; { - PyObject *cond = _tmp_1; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsTrue(cond)); } @@ -2844,10 +2962,12 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { - PyObject *_tmp_1 = stack_pointer[-1]; + PyObject *value; + PyObject *b; + PyObject *cond; + // IS_NONE + value = stack_pointer[-1]; { - PyObject *value = _tmp_1; - PyObject *b; if (Py_IsNone(value)) { b = Py_True; } @@ -2855,10 +2975,10 @@ b = Py_False; Py_DECREF(value); } - _tmp_1 = b; } + // POP_JUMP_IF_FALSE + cond = b; { - PyObject *cond = _tmp_1; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsFalse(cond)); } @@ -2877,8 +2997,9 @@ } TARGET(GET_LEN) { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *len_o; + obj = stack_pointer[-1]; // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; @@ -2890,10 +3011,13 @@ } TARGET(MATCH_CLASS) { - PyObject *names = stack_pointer[-1]; - PyObject *type = stack_pointer[-2]; - PyObject *subject = stack_pointer[-3]; + PyObject *names; + PyObject *type; + PyObject *subject; PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); @@ -2914,8 +3038,9 @@ } TARGET(MATCH_MAPPING) { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -2924,8 +3049,9 @@ } TARGET(MATCH_SEQUENCE) { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -2934,9 +3060,11 @@ } TARGET(MATCH_KEYS) { - PyObject *keys = stack_pointer[-1]; - PyObject *subject = stack_pointer[-2]; + PyObject *keys; + PyObject *subject; PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; @@ -2946,8 +3074,9 @@ } TARGET(GET_ITER) { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); @@ -2957,8 +3086,9 @@ } TARGET(GET_YIELD_FROM_ITER) { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -2990,8 +3120,9 @@ TARGET(FOR_ITER) { PREDICTED(FOR_ITER); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3059,15 +3190,15 @@ } TARGET(FOR_ITER_LIST) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; + PyObject *iter; + PyObject *next; + // _ITER_CHECK_LIST + iter = stack_pointer[-1]; { - PyObject *iter = _tmp_2; DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); - _tmp_2 = iter; } + // _ITER_JUMP_LIST { - PyObject *iter = _tmp_2; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); @@ -3084,37 +3215,32 @@ JUMPBY(oparg + 1); DISPATCH(); } - _tmp_2 = iter; } + // _ITER_NEXT_LIST { - PyObject *iter = _tmp_2; - PyObject *next; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - _tmp_2 = iter; - _tmp_1 = next; } - next_instr += 1; STACK_GROW(1); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_TUPLE) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; + PyObject *iter; + PyObject *next; + // _ITER_CHECK_TUPLE + iter = stack_pointer[-1]; { - PyObject *iter = _tmp_2; DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); - _tmp_2 = iter; } + // _ITER_JUMP_TUPLE { - PyObject *iter = _tmp_2; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); @@ -3131,38 +3257,33 @@ JUMPBY(oparg + 1); DISPATCH(); } - _tmp_2 = iter; } + // _ITER_NEXT_TUPLE { - PyObject *iter = _tmp_2; - PyObject *next; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - _tmp_2 = iter; - _tmp_1 = next; } - next_instr += 1; STACK_GROW(1); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_RANGE) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; + PyObject *iter; + PyObject *next; + // _ITER_CHECK_RANGE + iter = stack_pointer[-1]; { - PyObject *iter = _tmp_2; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - _tmp_2 = iter; } + // _ITER_JUMP_RANGE { - PyObject *iter = _tmp_2; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); @@ -3174,11 +3295,9 @@ JUMPBY(oparg + 1); DISPATCH(); } - _tmp_2 = iter; } + // _ITER_NEXT_RANGE { - PyObject *iter = _tmp_2; - PyObject *next; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); @@ -3187,18 +3306,16 @@ r->len--; next = PyLong_FromLong(value); if (next == NULL) goto error; - _tmp_2 = iter; - _tmp_1 = next; } - next_instr += 1; STACK_GROW(1); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_GEN) { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -3214,12 +3331,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); + STACK_GROW(1); } TARGET(BEFORE_ASYNC_WITH) { - PyObject *mgr = stack_pointer[-1]; + PyObject *mgr; PyObject *exit; PyObject *res; + mgr = stack_pointer[-1]; PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3250,15 +3369,16 @@ if (true) goto pop_1_error; } STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = exit; + stack_pointer[-1] = res; DISPATCH(); } TARGET(BEFORE_WITH) { - PyObject *mgr = stack_pointer[-1]; + PyObject *mgr; PyObject *exit; PyObject *res; + mgr = stack_pointer[-1]; /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3292,16 +3412,19 @@ if (true) goto pop_1_error; } STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = exit; + stack_pointer[-1] = res; DISPATCH(); } TARGET(WITH_EXCEPT_START) { - PyObject *val = stack_pointer[-1]; - PyObject *lasti = stack_pointer[-3]; - PyObject *exit_func = stack_pointer[-4]; + PyObject *val; + PyObject *lasti; + PyObject *exit_func; PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3333,8 +3456,9 @@ } TARGET(PUSH_EXC_INFO) { - PyObject *new_exc = stack_pointer[-1]; + PyObject *new_exc; PyObject *prev_exc; + new_exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3345,15 +3469,16 @@ assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); STACK_GROW(1); - stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; + stack_pointer[-1] = new_exc; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; + PyObject *self; + PyObject *res2; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); @@ -3374,16 +3499,17 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; + PyObject *self; + PyObject *res2; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); @@ -3396,15 +3522,16 @@ res2 = Py_NewRef(descr); res = self; STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - PyObject *self = stack_pointer[-1]; + PyObject *self; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); @@ -3428,8 +3555,9 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - PyObject *self = stack_pointer[-1]; + PyObject *self; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert((oparg & 1) == 0); @@ -3447,9 +3575,10 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; + PyObject *self; + PyObject *res2; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); @@ -3466,8 +3595,8 @@ res2 = Py_NewRef(descr); res = self; STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 9; DISPATCH(); } @@ -3497,10 +3626,13 @@ TARGET(CALL) { PREDICTED(CALL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3591,8 +3723,10 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *callable; + PyObject *method; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3602,13 +3736,18 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&next_instr[1].cache); ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -3635,12 +3774,17 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_PY_WITH_DEFAULTS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&next_instr[1].cache); ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -3677,13 +3821,18 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_NO_KW_TYPE_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3701,10 +3850,13 @@ } TARGET(CALL_NO_KW_STR_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3724,10 +3876,13 @@ } TARGET(CALL_NO_KW_TUPLE_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3747,9 +3902,12 @@ } TARGET(CALL_NO_KW_ALLOC_AND_ENTER_INIT) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) @@ -3800,10 +3958,13 @@ * as it will be checked after start_frame */ tstate->py_recursion_remaining--; goto start_frame; + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(EXIT_INIT_CHECK) { - PyObject *should_be_none = stack_pointer[-1]; + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -3816,10 +3977,13 @@ } TARGET(CALL_BUILTIN_CLASS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3850,10 +4014,13 @@ } TARGET(CALL_NO_KW_BUILTIN_O) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -3890,10 +4057,13 @@ } TARGET(CALL_NO_KW_BUILTIN_FAST) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -3934,10 +4104,13 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3978,10 +4151,13 @@ } TARGET(CALL_NO_KW_LEN) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* len(o) */ int is_meth = method != NULL; @@ -3992,7 +4168,7 @@ total_args++; } DEOPT_IF(total_args != 1, CALL); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); PyObject *arg = args[0]; @@ -4014,10 +4190,13 @@ } TARGET(CALL_NO_KW_ISINSTANCE) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4028,7 +4207,7 @@ total_args++; } DEOPT_IF(total_args != 2, CALL); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); PyObject *cls = args[1]; @@ -4052,13 +4231,16 @@ } TARGET(CALL_NO_KW_LIST_APPEND) { - PyObject **args = (stack_pointer - oparg); - PyObject *self = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *self; + PyObject *method; + args = stack_pointer - oparg; + self = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); assert(method != NULL); - PyInterpreterState *interp = _PyInterpreterState_GET(); + PyInterpreterState *interp = tstate->interp; DEOPT_IF(method != interp->callable_cache.list_append, CALL); DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); @@ -4072,12 +4254,16 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -4117,9 +4303,11 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4157,9 +4345,11 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4197,9 +4387,11 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -4241,10 +4433,13 @@ TARGET(CALL_FUNCTION_EX) { PREDICTED(CALL_FUNCTION_EX); - PyObject *kwargs = (oparg & 1) ? stack_pointer[-(((oparg & 1) ? 1 : 0))] : NULL; - PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; - PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; + PyObject *kwargs = NULL; + PyObject *callargs; + PyObject *func; PyObject *result; + if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } + callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; + func = stack_pointer[-2 - (oparg & 1 ? 1 : 0)]; // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4319,8 +4514,9 @@ } TARGET(MAKE_FUNCTION) { - PyObject *codeobj = stack_pointer[-1]; + PyObject *codeobj; PyObject *func; + codeobj = stack_pointer[-1]; PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4337,8 +4533,10 @@ } TARGET(SET_FUNCTION_ATTRIBUTE) { - PyObject *func = stack_pointer[-1]; - PyObject *attr = stack_pointer[-2]; + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -4392,10 +4590,13 @@ } TARGET(BUILD_SLICE) { - PyObject *step = (oparg == 3) ? stack_pointer[-(((oparg == 3) ? 1 : 0))] : NULL; - PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; - PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; + PyObject *step = NULL; + PyObject *stop; + PyObject *start; PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } + stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; + start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); @@ -4408,8 +4609,9 @@ } TARGET(CONVERT_VALUE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *result; + value = stack_pointer[-1]; convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; @@ -4421,8 +4623,9 @@ } TARGET(FORMAT_SIMPLE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -4438,9 +4641,11 @@ } TARGET(FORMAT_WITH_SPEC) { - PyObject *fmt_spec = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *fmt_spec; + PyObject *value; PyObject *res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); @@ -4451,8 +4656,9 @@ } TARGET(COPY) { - PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; + PyObject *bottom; PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = Py_NewRef(bottom); STACK_GROW(1); @@ -4463,9 +4669,11 @@ TARGET(BINARY_OP) { PREDICTED(BINARY_OP); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *rhs = stack_pointer[-1]; - PyObject *lhs = stack_pointer[-2]; + PyObject *rhs; + PyObject *lhs; PyObject *res; + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4490,11 +4698,13 @@ } TARGET(SWAP) { - PyObject *top = stack_pointer[-1]; - PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; stack_pointer[-1] = bottom; - stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } diff --git a/Python/hamt.c b/Python/hamt.c index c78b5a7fab94f0..24265edc2c3fd4 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -514,7 +514,7 @@ hamt_node_bitmap_new(Py_ssize_t size) /* Since bitmap nodes are immutable, we can cache the instance for size=0 and reuse it whenever we need an empty bitmap node. */ - return (PyHamtNode *)Py_NewRef(&_Py_SINGLETON(hamt_bitmap_node_empty)); + return (PyHamtNode *)&_Py_SINGLETON(hamt_bitmap_node_empty); } assert(size >= 0); diff --git a/Python/import.c b/Python/import.c index e381d1fd7d433e..56b2dc1a4ada2c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -601,7 +601,7 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) when an extension is loaded. This includes when it is imported for the first time. - Here's a summary, using importlib._boostrap._load() as a starting point. + Here's a summary, using importlib._bootstrap._load() as a starting point. 1. importlib._bootstrap._load() 2. _load(): acquire import lock @@ -1073,6 +1073,7 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) However, this decref would be problematic if the module def were dynamically allocated, it were the last ref, and this function were called with an interpreter other than the def's owner. */ + assert(_Py_IsImmortal(entry->value)); entry->value = NULL; finally: diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py deleted file mode 100755 index 5843079b729936..00000000000000 --- a/Python/makeopcodetargets.py +++ /dev/null @@ -1,56 +0,0 @@ -#! /usr/bin/env python -"""Generate C code for the jump table of the threaded code interpreter -(for compilers supporting computed gotos or "labels-as-values", such as gcc). -""" - -import os -import sys - - -# 2023-04-27(warsaw): Pre-Python 3.12, this would catch ImportErrors and try to -# import imp, and then use imp.load_module(). The imp module was removed in -# Python 3.12 (and long deprecated before that), and it's unclear under what -# conditions this import will now fail, so the fallback was simply removed. -from importlib.machinery import SourceFileLoader - -def find_module(modname): - """Finds and returns a module in the local dist/checkout. - """ - modpath = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "Lib", modname + ".py") - return SourceFileLoader(modname, modpath).load_module() - - -def write_contents(f): - """Write C code contents to the target file object. - """ - opcode = find_module('opcode') - _opcode_metadata = find_module('_opcode_metadata') - targets = ['_unknown_opcode'] * 256 - for opname, op in opcode.opmap.items(): - if not opcode.is_pseudo(op): - targets[op] = "TARGET_%s" % opname - next_op = 1 - for opname in _opcode_metadata._specialized_instructions: - while targets[next_op] != '_unknown_opcode': - next_op += 1 - targets[next_op] = "TARGET_%s" % opname - f.write("static void *opcode_targets[256] = {\n") - f.write(",\n".join([" &&%s" % s for s in targets])) - f.write("\n};\n") - - -def main(): - if len(sys.argv) >= 3: - sys.exit("Too many arguments") - if len(sys.argv) == 2: - target = sys.argv[1] - else: - target = "Python/opcode_targets.h" - with open(target, "w") as f: - write_contents(f) - print("Jump table written into %s" % target) - - -if __name__ == "__main__": - main() diff --git a/Python/optimizer.c b/Python/optimizer.c index 09120c33d130ca..d2ed8dfcd2cff2 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -320,13 +320,7 @@ uop_name(int index) { static Py_ssize_t uop_len(_PyUOpExecutorObject *self) { - int count = 0; - for (; count < _Py_UOP_MAX_TRACE_LENGTH; count++) { - if (self->trace[count].opcode == 0) { - break; - } - } - return count; + return Py_SIZE(self); } static PyObject * @@ -368,8 +362,8 @@ PySequenceMethods uop_as_sequence = { static PyTypeObject UOpExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "uop_executor", - .tp_basicsize = sizeof(_PyUOpExecutorObject), - .tp_itemsize = 0, + .tp_basicsize = sizeof(_PyUOpExecutorObject) - sizeof(_PyUOpInstruction), + .tp_itemsize = sizeof(_PyUOpInstruction), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_dealloc = (destructor)uop_dealloc, .tp_as_sequence = &uop_as_sequence, @@ -579,7 +573,8 @@ translate_bytecode_to_trace( for (int i = 0; i < nuops; i++) { oparg = orig_oparg; uint64_t operand = 0; - int offset = expansion->uops[i].offset; + // Add one to account for the actual opcode/oparg pair: + int offset = expansion->uops[i].offset + 1; switch (expansion->uops[i].size) { case OPARG_FULL: if (extras && OPCODE_HAS_JUMP(opcode)) { @@ -698,15 +693,12 @@ uop_optimize( return trace_length; } OBJECT_STAT_INC(optimization_traces_created); - _PyUOpExecutorObject *executor = PyObject_New(_PyUOpExecutorObject, &UOpExecutor_Type); + _PyUOpExecutorObject *executor = PyObject_NewVar(_PyUOpExecutorObject, &UOpExecutor_Type, trace_length); if (executor == NULL) { return -1; } executor->base.execute = _PyUopExecute; memcpy(executor->trace, trace, trace_length * sizeof(_PyUOpInstruction)); - if (trace_length < _Py_UOP_MAX_TRACE_LENGTH) { - executor->trace[trace_length].opcode = 0; // Sentinel - } *exec_ptr = (_PyExecutorObject *)executor; return 1; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ceca7776a276ed..7a17f92b550c05 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1921,11 +1921,11 @@ Py_FinalizeEx(void) } if (dump_refs) { - _Py_PrintReferences(stderr); + _Py_PrintReferences(tstate->interp, stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferences(dump_refs_fp); + _Py_PrintReferences(tstate->interp, dump_refs_fp); } #endif /* Py_TRACE_REFS */ @@ -1961,11 +1961,11 @@ Py_FinalizeEx(void) */ if (dump_refs) { - _Py_PrintReferenceAddresses(stderr); + _Py_PrintReferenceAddresses(tstate->interp, stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferenceAddresses(dump_refs_fp); + _Py_PrintReferenceAddresses(tstate->interp, dump_refs_fp); fclose(dump_refs_fp); } #endif /* Py_TRACE_REFS */ diff --git a/Python/specialize.c b/Python/specialize.c index 1669ce17fc804e..de329ef1195cbf 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -18,7 +18,8 @@ */ #ifdef Py_STATS -PyStats _py_stats_struct = { 0 }; +GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; +PyStats _py_stats_struct = { .gc_stats = &_py_gc_stats[0] }; PyStats *_py_stats = NULL; #define ADD_STAT_TO_DICT(res, field) \ @@ -202,17 +203,32 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->optimization_uops_executed); } +static void +print_gc_stats(FILE *out, GCStats *stats) +{ + for (int i = 0; i < NUM_GENERATIONS; i++) { + fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); + fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); + fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); + } +} + static void print_stats(FILE *out, PyStats *stats) { print_spec_stats(out, stats->opcode_stats); print_call_stats(out, &stats->call_stats); print_object_stats(out, &stats->object_stats); + print_gc_stats(out, stats->gc_stats); } void _Py_StatsClear(void) { + for (int i = 0; i < NUM_GENERATIONS; i++) { + _py_gc_stats[i] = (GCStats) { 0 }; + } _py_stats_struct = (PyStats) { 0 }; + _py_stats_struct.gc_stats = _py_gc_stats; } void diff --git a/Python/symtable.c b/Python/symtable.c index 04be3192d6c7c4..e9adbd5d29b1f9 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -282,17 +282,10 @@ symtable_new(void) return NULL; } -/* When compiling the use of C stack is probably going to be a lot - lighter than when executing Python code but still can overflow - and causing a Python crash if not checked (e.g. eval("()"*300000)). - Using the current recursion limit for the compiler seems too - restrictive (it caused at least one test to fail) so a factor is - used to allow deeper recursion when compiling an expression. - - Using a scaling factor means this should automatically adjust when +/* Using a scaling factor means this should automatically adjust when the recursion limit is adjusted for small or large C stack allocations. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 19e5eab822a235..3a817326c94cbb 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -64,6 +64,7 @@ def get_python_module_dict(filename): def main(opcode_py, _opcode_metadata_py='Lib/_opcode_metadata.py', outfile='Include/opcode.h', + opcode_targets_h='Python/opcode_targets.h', internaloutfile='Include/internal/pycore_opcode.h'): _opcode_metadata = get_python_module_dict(_opcode_metadata_py) @@ -71,11 +72,7 @@ def main(opcode_py, opcode = get_python_module_dict(opcode_py) opmap = opcode['opmap'] opname = opcode['opname'] - is_pseudo = opcode['is_pseudo'] - ENABLE_SPECIALIZATION = opcode["ENABLE_SPECIALIZATION"] - MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] - MAX_PSEUDO_OPCODE = opcode["MAX_PSEUDO_OPCODE"] MIN_INSTRUMENTED_OPCODE = opcode["MIN_INSTRUMENTED_OPCODE"] NUM_OPCODES = len(opname) @@ -101,16 +98,11 @@ def main(opcode_py, for name in opname: if name in opmap: op = opmap[name] - if op == MIN_PSEUDO_OPCODE: - fobj.write(DEFINE.format("MIN_PSEUDO_OPCODE", MIN_PSEUDO_OPCODE)) if op == MIN_INSTRUMENTED_OPCODE: fobj.write(DEFINE.format("MIN_INSTRUMENTED_OPCODE", MIN_INSTRUMENTED_OPCODE)) fobj.write(DEFINE.format(name, op)) - if op == MAX_PSEUDO_OPCODE: - fobj.write(DEFINE.format("MAX_PSEUDO_OPCODE", MAX_PSEUDO_OPCODE)) - for name, op in specialized_opmap.items(): fobj.write(DEFINE.format(name, op)) @@ -126,7 +118,7 @@ def main(opcode_py, deoptcodes = {} for basic, op in opmap.items(): - if not is_pseudo(op): + if op < 256: deoptcodes[basic] = basic for basic, family in _opcode_metadata["_specializations"].items(): for specialized in family: @@ -141,10 +133,6 @@ def main(opcode_py, for i, (op, _) in enumerate(opcode["_nb_ops"]): fobj.write(DEFINE.format(op, i)) - fobj.write("\n") - fobj.write("/* Defined in Lib/opcode.py */\n") - fobj.write(f"#define ENABLE_SPECIALIZATION {int(ENABLE_SPECIALIZATION)}") - iobj.write("\n") iobj.write(f"\nextern const char *const _PyOpcode_OpName[{NUM_OPCODES}];\n") iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") @@ -166,9 +154,18 @@ def main(opcode_py, fobj.write(footer) iobj.write(internal_footer) + with open(opcode_targets_h, "w") as f: + targets = ["_unknown_opcode"] * 256 + for op, name in enumerate(opname_including_specialized): + if op < 256 and not name.startswith("<"): + targets[op] = f"TARGET_{name}" + + f.write("static void *opcode_targets[256] = {\n") + f.write(",\n".join([f" &&{s}" for s in targets])) + f.write("\n};\n") print(f"{outfile} regenerated from {opcode_py}") if __name__ == '__main__': - main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) + main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5]) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index cf6d3b24435238..099f20b5a1b433 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -213,6 +213,7 @@ Modules/_decimal/_decimal.c - invalid_rounding_err - Modules/_decimal/_decimal.c - invalid_signals_err - Modules/_decimal/_decimal.c - signal_map_template - Modules/_decimal/_decimal.c - ssize_constants - +Modules/_decimal/_decimal.c - INVALID_SIGNALDICT_ERROR_MSG - Modules/_elementtree.c - ExpatMemoryHandler - Modules/_hashopenssl.c - py_hashes - Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 53ae7360b8ae12..2db1cd01c19ae5 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -13,7 +13,6 @@ MacroParts, OverriddenInstructionPlaceHolder, PseudoInstruction, - StackEffectMapping, ) import parsing from parsing import StackEffect @@ -34,11 +33,12 @@ class Analyzer: input_filenames: list[str] errors: int = 0 + warnings: int = 0 def __init__(self, input_filenames: list[str]): self.input_filenames = input_filenames - def error(self, msg: str, node: parsing.Node) -> None: + def message(self, msg: str, node: parsing.Node) -> None: lineno = 0 filename = "" if context := node.context: @@ -49,8 +49,18 @@ def error(self, msg: str, node: parsing.Node) -> None: if token.kind != "COMMENT": break print(f"{filename}:{lineno}: {msg}", file=sys.stderr) + + def error(self, msg: str, node: parsing.Node) -> None: + self.message("error: " + msg, node) self.errors += 1 + def warning(self, msg: str, node: parsing.Node) -> None: + self.message("warning: " + msg, node) + self.warnings += 1 + + def note(self, msg: str, node: parsing.Node) -> None: + self.message("note: " + msg, node) + everything: list[ parsing.InstDef | parsing.Macro @@ -83,8 +93,15 @@ def parse(self) -> None: self.parse_file(filename, instrs_idx) files = " + ".join(self.input_filenames) + n_instrs = 0 + n_ops = 0 + for instr in self.instrs.values(): + if instr.kind == "op": + n_ops += 1 + else: + n_instrs += 1 print( - f"Read {len(self.instrs)} instructions/ops, " + f"Read {n_instrs} instructions, {n_ops} ops, " f"{len(self.macros)} macros, {len(self.pseudos)} pseudos, " f"and {len(self.families)} families from {files}", file=sys.stderr, @@ -270,14 +287,72 @@ def analyze_macros_and_pseudos(self) -> None: self.macro_instrs = {} self.pseudo_instrs = {} for name, macro in self.macros.items(): - self.macro_instrs[name] = self.analyze_macro(macro) + self.macro_instrs[name] = mac = self.analyze_macro(macro) + self.check_macro_consistency(mac) for name, pseudo in self.pseudos.items(): self.pseudo_instrs[name] = self.analyze_pseudo(pseudo) + # TODO: Merge with similar code in stacking.py, write_components() + def check_macro_consistency(self, mac: MacroInstruction) -> None: + def get_var_names(instr: Instruction) -> dict[str, StackEffect]: + vars: dict[str, StackEffect] = {} + for eff in instr.input_effects + instr.output_effects: + if eff.name == UNUSED: + continue + if eff.name in vars: + if vars[eff.name] != eff: + self.error( + f"Instruction {instr.name!r} has " + f"inconsistent type/cond/size for variable " + f"{eff.name!r}: {vars[eff.name]} vs {eff}", + instr.inst, + ) + else: + vars[eff.name] = eff + return vars + + all_vars: dict[str, StackEffect] = {} + # print("Checking", mac.name) + prevop: Instruction | None = None + for part in mac.parts: + if not isinstance(part, Component): + continue + vars = get_var_names(part.instr) + # print(" //", part.instr.name, "//", vars) + for name, eff in vars.items(): + if name in all_vars: + if all_vars[name] != eff: + self.error( + f"Macro {mac.name!r} has " + f"inconsistent type/cond/size for variable " + f"{name!r}: " + f"{all_vars[name]} vs {eff} in {part.instr.name!r}", + mac.macro, + ) + else: + all_vars[name] = eff + if prevop is not None: + pushes = list(prevop.output_effects) + pops = list(reversed(part.instr.input_effects)) + copies: list[tuple[StackEffect, StackEffect]] = [] + while pushes and pops and pushes[-1] == pops[0]: + src, dst = pushes.pop(), pops.pop(0) + if src.name == dst.name or dst.name == UNUSED: + continue + copies.append((src, dst)) + reads = set(copy[0].name for copy in copies) + writes = set(copy[1].name for copy in copies) + if reads & writes: + self.error( + f"Macro {mac.name!r} has conflicting copies " + f"(source of one copy is destination of another): " + f"{reads & writes}", + mac.macro, + ) + prevop = part.instr + def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: components = self.check_macro_components(macro) - stack, initial_sp = self.stack_analysis(components) - sp = initial_sp parts: MacroParts = [] flags = InstructionFlags.newEmpty() offset = 0 @@ -287,20 +362,15 @@ def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: parts.append(ceffect) offset += ceffect.size case Instruction() as instr: - part, sp, offset = self.analyze_instruction( - instr, stack, sp, offset - ) + part, offset = self.analyze_instruction(instr, offset) parts.append(part) flags.add(instr.instr_flags) case _: typing.assert_never(component) - final_sp = sp format = "IB" if offset: format += "C" + "0" * (offset - 1) - return MacroInstruction( - macro.name, stack, initial_sp, final_sp, format, flags, macro, parts, offset - ) + return MacroInstruction(macro.name, format, flags, macro, parts, offset) def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: targets = [self.instrs[target] for target in pseudo.targets] @@ -312,24 +382,15 @@ def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: return PseudoInstruction(pseudo.name, targets, fmts[0], targets[0].instr_flags) def analyze_instruction( - self, instr: Instruction, stack: list[StackEffect], sp: int, offset: int - ) -> tuple[Component, int, int]: - input_mapping: StackEffectMapping = [] - for ieffect in reversed(instr.input_effects): - sp -= 1 - input_mapping.append((stack[sp], ieffect)) - output_mapping: StackEffectMapping = [] - for oeffect in instr.output_effects: - output_mapping.append((stack[sp], oeffect)) - sp += 1 + self, instr: Instruction, offset: int + ) -> tuple[Component, int]: active_effects: list[ActiveCacheEffect] = [] for ceffect in instr.cache_effects: if ceffect.name != UNUSED: active_effects.append(ActiveCacheEffect(ceffect, offset)) offset += ceffect.size return ( - Component(instr, input_mapping, output_mapping, active_effects), - sp, + Component(instr, active_effects), offset, ) @@ -348,65 +409,3 @@ def check_macro_components( case _: typing.assert_never(uop) return components - - def stack_analysis( - self, components: typing.Iterable[InstructionOrCacheEffect] - ) -> tuple[list[StackEffect], int]: - """Analyze a macro. - - Ignore cache effects. - - Return the list of variables (as StackEffects) and the initial stack pointer. - """ - lowest = current = highest = 0 - conditions: dict[int, str] = {} # Indexed by 'current'. - last_instr: Instruction | None = None - for thing in components: - if isinstance(thing, Instruction): - last_instr = thing - for thing in components: - match thing: - case Instruction() as instr: - if any( - eff.size for eff in instr.input_effects + instr.output_effects - ): - # TODO: Eventually this will be needed, at least for macros. - self.error( - f"Instruction {instr.name!r} has variable-sized stack effect, " - "which are not supported in macro instructions", - instr.inst, # TODO: Pass name+location of macro - ) - if any(eff.cond for eff in instr.input_effects): - self.error( - f"Instruction {instr.name!r} has conditional input stack effect, " - "which are not supported in macro instructions", - instr.inst, # TODO: Pass name+location of macro - ) - if ( - any(eff.cond for eff in instr.output_effects) - and instr is not last_instr - ): - self.error( - f"Instruction {instr.name!r} has conditional output stack effect, " - "but is not the last instruction in a macro", - instr.inst, # TODO: Pass name+location of macro - ) - current -= len(instr.input_effects) - lowest = min(lowest, current) - for eff in instr.output_effects: - if eff.cond: - conditions[current] = eff.cond - current += 1 - highest = max(highest, current) - case parsing.CacheEffect(): - pass - case _: - typing.assert_never(thing) - # At this point, 'current' is the net stack effect, - # and 'lowest' and 'highest' are the extremes. - # Note that 'lowest' may be negative. - stack = [ - StackEffect(f"_tmp_{i}", "", conditions.get(highest - i, "")) - for i in reversed(range(1, highest - lowest + 1)) - ] - return stack, -lowest diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index 78acd93b73ac31..f7ebdeb0d65677 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -49,9 +49,9 @@ def add(self, other: "InstructionFlags") -> None: if value: setattr(self, name, value) - def names(self, value=None): + def names(self, value=None) -> list[str]: if value is None: - return dataclasses.asdict(self).keys() + return list(dataclasses.asdict(self).keys()) return [n for n, v in dataclasses.asdict(self).items() if v == value] def bitmap(self) -> int: diff --git a/Tools/cases_generator/formatting.py b/Tools/cases_generator/formatting.py index 6b5a375af891be..5894751bd9635c 100644 --- a/Tools/cases_generator/formatting.py +++ b/Tools/cases_generator/formatting.py @@ -2,7 +2,7 @@ import re import typing -from parsing import StackEffect +from parsing import StackEffect, Family UNUSED = "unused" @@ -19,8 +19,11 @@ class Formatter: nominal_filename: str def __init__( - self, stream: typing.TextIO, indent: int, - emit_line_directives: bool = False, comment: str = "//", + self, + stream: typing.TextIO, + indent: int, + emit_line_directives: bool = False, + comment: str = "//", ) -> None: self.stream = stream self.prefix = " " * indent @@ -93,8 +96,11 @@ def declare(self, dst: StackEffect, src: StackEffect | None): typ = f"{dst.type}" if dst.type else "PyObject *" if src: cast = self.cast(dst, src) - init = f" = {cast}{src.name}" - elif dst.cond: + initexpr = f"{cast}{src.name}" + if src.cond and src.cond != "1": + initexpr = f"{parenthesize_cond(src.cond)} ? {initexpr} : NULL" + init = f" = {initexpr}" + elif dst.cond and dst.cond != "1": init = " = NULL" else: init = "" @@ -102,10 +108,7 @@ def declare(self, dst: StackEffect, src: StackEffect | None): self.emit(f"{typ}{sepa}{dst.name}{init};") def assign(self, dst: StackEffect, src: StackEffect): - if src.name == UNUSED: - return - if src.size: - # Don't write sized arrays -- it's up to the user code. + if src.name == UNUSED or dst.name == UNUSED: return cast = self.cast(dst, src) if re.match(r"^REG\(oparg(\d+)\)$", dst.name): @@ -122,6 +125,23 @@ def assign(self, dst: StackEffect, src: StackEffect): def cast(self, dst: StackEffect, src: StackEffect) -> str: return f"({dst.type or 'PyObject *'})" if src.type != dst.type else "" + def static_assert_family_size( + self, name: str, family: Family | None, cache_offset: int + ) -> None: + """Emit a static_assert for the size of a family, if known. + + This will fail at compile time if the cache size computed from + the instruction definition does not match the size of the struct + used by specialize.c. + """ + if family and name == family.name: + cache_size = family.size + if cache_size: + self.emit( + f"static_assert({cache_size} == {cache_offset}, " + f'"incorrect cache size");' + ) + def prettify_filename(filename: str) -> str: # Make filename more user-friendly and less platform-specific, @@ -178,11 +198,8 @@ def maybe_parenthesize(sym: str) -> str: return f"({sym})" -def string_effect_size(arg: tuple[int, str]) -> str: - numeric, symbolic = arg - if numeric and symbolic: - return f"{numeric} + {symbolic}" - elif symbolic: - return symbolic - else: - return str(numeric) +def parenthesize_cond(cond: str) -> str: + """Parenthesize a condition, but only if it contains ?: itself.""" + if "?" in cond: + cond = f"({cond})" + return cond diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 967e1e2f5b63bb..d35a16a80e8d00 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -4,14 +4,14 @@ """ import argparse -import contextlib import os import posixpath import sys import typing +import stacking # Early import to avoid circular import from analysis import Analyzer -from formatting import Formatter, list_effect_size, maybe_parenthesize +from formatting import Formatter, list_effect_size from flags import InstructionFlags, variable_used from instructions import ( AnyInstruction, @@ -118,41 +118,7 @@ def effect_str(effects: list[StackEffect]) -> str: pushed = "" case parsing.Macro(): instr = self.macro_instrs[thing.name] - parts = [comp for comp in instr.parts if isinstance(comp, Component)] - # Note: stack_analysis() already verifies that macro components - # have no variable-sized stack effects. - low = 0 - sp = 0 - high = 0 - pushed_symbolic: list[str] = [] - for comp in parts: - for effect in comp.instr.input_effects: - assert not effect.cond, effect - assert not effect.size, effect - sp -= 1 - low = min(low, sp) - for effect in comp.instr.output_effects: - assert not effect.size, effect - if effect.cond: - if effect.cond in ("0", "1"): - pushed_symbolic.append(effect.cond) - else: - pushed_symbolic.append( - maybe_parenthesize( - f"{maybe_parenthesize(effect.cond)} ? 1 : 0" - ) - ) - sp += 1 - high = max(sp, high) - if high != max(0, sp): - # If you get this, intermediate stack growth occurs, - # and stack size calculations may go awry. - # E.g. [push, pop]. The fix would be for stack size - # calculations to use the micro ops. - self.error("Macro has virtual stack growth", thing) - popped = str(-low) - pushed_symbolic.append(str(sp - low - len(pushed_symbolic))) - pushed = " + ".join(pushed_symbolic) + popped, pushed = stacking.get_stack_effect_info_for_macro(instr) case parsing.Pseudo(): instr = self.pseudo_instrs[thing.name] popped = pushed = None @@ -258,7 +224,8 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case _: typing.assert_never(thing) all_formats.add(format) - # Turn it into a list of enum definitions. + + # Turn it into a sorted list of enum values. format_enums = [INSTR_FMT_PREFIX + format for format in sorted(all_formats)] with open(metadata_filename, "w") as f: @@ -276,8 +243,10 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_stack_effect_functions() - # Write type definitions - self.out.emit(f"enum InstructionFormat {{ {', '.join(format_enums)} }};") + # Write the enum definition for instruction formats. + with self.out.block("enum InstructionFormat", ";"): + for enum in format_enums: + self.out.emit(enum + ",") self.out.emit("") self.out.emit( @@ -374,7 +343,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No # Since an 'op' is not a bytecode, it has no expansion; but 'inst' is if instr.kind == "inst" and instr.is_viable_uop(): # Construct a dummy Component -- input/output mappings are not used - part = Component(instr, [], [], instr.active_caches) + part = Component(instr, instr.active_caches) self.write_macro_expansions(instr.name, [part]) elif instr.kind == "inst" and variable_used( instr.inst, "oparg1" @@ -468,7 +437,15 @@ def write_macro_expansions(self, name: str, parts: MacroParts) -> None: if isinstance(part, Component): # All component instructions must be viable uops if not part.instr.is_viable_uop(): - print(f"NOTE: Part {part.instr.name} of {name} is not a viable uop") + # This note just reminds us about macros that cannot + # be expanded to Tier 2 uops. It is not an error. + # It is sometimes emitted for macros that have a + # manual translation in translate_bytecode_to_trace() + # in Python/optimizer.c. + self.note( + f"Part {part.instr.name} of {name} is not a viable uop", + part.instr.inst, + ) return if not part.active_caches: size, offset = OPARG_SIZES["OPARG_FULL"], 0 @@ -512,7 +489,7 @@ def write_super_expansions(self, name: str) -> None: instr2 = self.instrs[name2] assert not instr1.active_caches, f"{name1} has active caches" assert not instr2.active_caches, f"{name2} has active caches" - expansions = [ + expansions: list[tuple[str, int, int]] = [ (name1, OPARG_SIZES["OPARG_TOP"], 0), (name2, OPARG_SIZES["OPARG_BOTTOM"], 0), ] @@ -563,7 +540,6 @@ def write_instructions( # Write and count instructions of all kinds n_instrs = 0 n_macros = 0 - n_pseudos = 0 for thing in self.everything: match thing: case OverriddenInstructionPlaceHolder(): @@ -574,15 +550,17 @@ def write_instructions( self.write_instr(self.instrs[thing.name]) case parsing.Macro(): n_macros += 1 - self.write_macro(self.macro_instrs[thing.name]) + mac = self.macro_instrs[thing.name] + stacking.write_macro_instr(mac, self.out, self.families.get(mac.name)) + # self.write_macro(self.macro_instrs[thing.name]) case parsing.Pseudo(): - n_pseudos += 1 + pass case _: typing.assert_never(thing) print( - f"Wrote {n_instrs} instructions, {n_macros} macros, " - f"and {n_pseudos} pseudos to {output_filename}", + f"Wrote {n_instrs} instructions and {n_macros} macros " + f"to {output_filename}", file=sys.stderr, ) @@ -590,6 +568,8 @@ def write_executor_instructions( self, executor_filename: str, emit_line_directives: bool ) -> None: """Generate cases for the Tier 2 interpreter.""" + n_instrs = 0 + n_uops = 0 with open(executor_filename, "w") as f: self.out = Formatter(f, 8, emit_line_directives) self.write_provenance_header() @@ -601,6 +581,10 @@ def write_executor_instructions( case parsing.InstDef(): instr = self.instrs[thing.name] if instr.is_viable_uop(): + if instr.kind == "op": + n_uops += 1 + else: + n_instrs += 1 self.out.emit("") with self.out.block(f"case {thing.name}:"): instr.write(self.out, tier=TIER_TWO) @@ -616,7 +600,7 @@ def write_executor_instructions( case _: typing.assert_never(thing) print( - f"Wrote some stuff to {executor_filename}", + f"Wrote {n_instrs} instructions and {n_uops} ops to {executor_filename}", file=sys.stderr, ) @@ -642,69 +626,6 @@ def write_instr(self, instr: Instruction) -> None: self.out.emit("CHECK_EVAL_BREAKER();") self.out.emit(f"DISPATCH();") - def write_macro(self, mac: MacroInstruction) -> None: - """Write code for a macro instruction.""" - last_instr: Instruction | None = None - with self.wrap_macro(mac): - cache_adjust = 0 - for part in mac.parts: - match part: - case parsing.CacheEffect(size=size): - cache_adjust += size - case Component() as comp: - last_instr = comp.instr - comp.write_body(self.out) - cache_adjust += comp.instr.cache_offset - - if cache_adjust: - self.out.emit(f"next_instr += {cache_adjust};") - - if ( - (family := self.families.get(mac.name)) - and mac.name == family.name - and (cache_size := family.size) - ): - self.out.emit( - f"static_assert({cache_size} == " - f'{cache_adjust}, "incorrect cache size");' - ) - - @contextlib.contextmanager - def wrap_macro(self, mac: MacroInstruction): - """Boilerplate for macro instructions.""" - # TODO: Somewhere (where?) make it so that if one instruction - # has an output that is input to another, and the variable names - # and types match and don't conflict with other instructions, - # that variable is declared with the right name and type in the - # outer block, rather than trusting the compiler to optimize it. - self.out.emit("") - with self.out.block(f"TARGET({mac.name})"): - if mac.predicted: - self.out.emit(f"PREDICTED({mac.name});") - - # The input effects should have no conditionals. - # Only the output effects do (for now). - ieffects = [ - StackEffect(eff.name, eff.type) if eff.cond else eff - for eff in mac.stack - ] - - for i, var in reversed(list(enumerate(ieffects))): - src = None - if i < mac.initial_sp: - src = StackEffect(f"stack_pointer[-{mac.initial_sp - i}]", "") - self.out.declare(var, src) - - yield - - self.out.stack_adjust(ieffects[: mac.initial_sp], mac.stack[: mac.final_sp]) - - for i, var in enumerate(reversed(mac.stack[: mac.final_sp]), 1): - dst = StackEffect(f"stack_pointer[-{i}]", "") - self.out.assign(dst, var) - - self.out.emit(f"DISPATCH();") - def main(): """Parse command line, parse input, analyze, write output.""" diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 6f42699d900b46..aa94dbb07ea1c0 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -2,17 +2,16 @@ import re import typing -from flags import InstructionFlags, variable_used_unspecialized +from flags import InstructionFlags, variable_used, variable_used_unspecialized from formatting import ( Formatter, UNUSED, - string_effect_size, list_effect_size, - maybe_parenthesize, ) import lexer as lx import parsing from parsing import StackEffect +import stacking BITS_PER_CODE_UNIT = 16 @@ -61,6 +60,7 @@ class Instruction: # Computed by constructor always_exits: bool + has_deopt: bool cache_offset: int cache_effects: list[parsing.CacheEffect] input_effects: list[StackEffect] @@ -83,6 +83,7 @@ def __init__(self, inst: parsing.InstDef): self.block ) self.always_exits = always_exits(self.block_text) + self.has_deopt = variable_used(self.inst, "DEOPT_IF") self.cache_effects = [ effect for effect in inst.inputs if isinstance(effect, parsing.CacheEffect) ] @@ -93,7 +94,7 @@ def __init__(self, inst: parsing.InstDef): self.output_effects = inst.outputs # For consistency/completeness unmoved_names: set[str] = set() for ieffect, oeffect in zip(self.input_effects, self.output_effects): - if ieffect.name == oeffect.name: + if ieffect == oeffect and ieffect.name == oeffect.name: unmoved_names.add(ieffect.name) else: break @@ -141,84 +142,17 @@ def is_viable_uop(self) -> bool: def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: """Write one instruction, sans prologue and epilogue.""" + # Write a static assertion that a family's cache size is correct - if family := self.family: - if self.name == family.name: - if cache_size := family.size: - out.emit( - f"static_assert({cache_size} == " - f'{self.cache_offset}, "incorrect cache size");' - ) + out.static_assert_family_size(self.name, self.family, self.cache_offset) # Write input stack effect variable declarations and initializations - ieffects = list(reversed(self.input_effects)) - for i, ieffect in enumerate(ieffects): - isize = string_effect_size( - list_effect_size([ieff for ieff in ieffects[: i + 1]]) - ) - if ieffect.size: - src = StackEffect( - f"(stack_pointer - {maybe_parenthesize(isize)})", "PyObject **" - ) - elif ieffect.cond: - src = StackEffect( - f"({ieffect.cond}) ? stack_pointer[-{maybe_parenthesize(isize)}] : NULL", - "", - ) - else: - src = StackEffect(f"stack_pointer[-{maybe_parenthesize(isize)}]", "") - out.declare(ieffect, src) - - # Write output stack effect variable declarations - isize = string_effect_size(list_effect_size(self.input_effects)) - input_names = {ieffect.name for ieffect in self.input_effects} - for i, oeffect in enumerate(self.output_effects): - if oeffect.name not in input_names: - if oeffect.size: - osize = string_effect_size( - list_effect_size([oeff for oeff in self.output_effects[:i]]) - ) - offset = "stack_pointer" - if isize != osize: - if isize != "0": - offset += f" - ({isize})" - if osize != "0": - offset += f" + {osize}" - src = StackEffect(offset, "PyObject **") - out.declare(oeffect, src) - else: - out.declare(oeffect, None) - - # out.emit(f"next_instr += OPSIZE({self.inst.name}) - 1;") - - self.write_body(out, 0, self.active_caches, tier=tier) + stacking.write_single_instr(self, out, tier) # Skip the rest if the block always exits if self.always_exits: return - # Write net stack growth/shrinkage - out.stack_adjust( - [ieff for ieff in self.input_effects], - [oeff for oeff in self.output_effects], - ) - - # Write output stack effect assignments - oeffects = list(reversed(self.output_effects)) - for i, oeffect in enumerate(oeffects): - if oeffect.name in self.unmoved_names: - continue - osize = string_effect_size( - list_effect_size([oeff for oeff in oeffects[: i + 1]]) - ) - if oeffect.size: - dst = StackEffect( - f"stack_pointer - {maybe_parenthesize(osize)}", "PyObject **" - ) - else: - dst = StackEffect(f"stack_pointer[-{maybe_parenthesize(osize)}]", "") - out.assign(dst, oeffect) - # Write cache effect if tier == TIER_ONE and self.cache_offset: out.emit(f"next_instr += {self.cache_offset};") @@ -274,7 +208,12 @@ def write_body( # These aren't DECREF'ed so they can stay. ieffs = list(self.input_effects) oeffs = list(self.output_effects) - while ieffs and oeffs and ieffs[0] == oeffs[0]: + while ( + ieffs + and oeffs + and ieffs[0] == oeffs[0] + and ieffs[0].name == oeffs[0].name + ): ieffs.pop(0) oeffs.pop(0) ninputs, symbolic = list_effect_size(ieffs) @@ -307,30 +246,13 @@ def write_body( InstructionOrCacheEffect = Instruction | parsing.CacheEffect -StackEffectMapping = list[tuple[StackEffect, StackEffect]] @dataclasses.dataclass class Component: instr: Instruction - input_mapping: StackEffectMapping - output_mapping: StackEffectMapping active_caches: list[ActiveCacheEffect] - def write_body(self, out: Formatter) -> None: - with out.block(""): - input_names = {ieffect.name for _, ieffect in self.input_mapping} - for var, ieffect in self.input_mapping: - out.declare(ieffect, var) - for _, oeffect in self.output_mapping: - if oeffect.name not in input_names: - out.declare(oeffect, None) - - self.instr.write_body(out, -4, self.active_caches) - - for var, oeffect in self.output_mapping: - out.assign(var, oeffect) - MacroParts = list[Component | parsing.CacheEffect] @@ -340,9 +262,6 @@ class MacroInstruction: """A macro instruction.""" name: str - stack: list[StackEffect] - initial_sp: int - final_sp: int instr_fmt: str instr_flags: InstructionFlags macro: parsing.Macro diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 290285dc03123c..5610ac661a8945 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -69,12 +69,18 @@ class Block(Node): @dataclass class StackEffect(Node): - name: str + name: str = field(compare=False) # __eq__ only uses type, cond, size type: str = "" # Optional `:type` cond: str = "" # Optional `if (cond)` size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond + def __repr__(self): + items = [self.name, self.type, self.cond, self.size] + while items and items[-1] == "": + del items[-1] + return f"StackEffect({', '.join(repr(item) for item in items)})" + @dataclass class Expression(Node): @@ -130,6 +136,7 @@ class Family(Node): size: str # Variable giving the cache size in code units members: list[str] + @dataclass class Pseudo(Node): name: str @@ -154,7 +161,13 @@ def inst_def(self) -> InstDef | None: if hdr := self.inst_header(): if block := self.block(): return InstDef( - hdr.override, hdr.register, hdr.kind, hdr.name, hdr.inputs, hdr.outputs, block + hdr.override, + hdr.register, + hdr.kind, + hdr.name, + hdr.inputs, + hdr.outputs, + block, ) raise self.make_syntax_error("Expected block") return None @@ -371,9 +384,7 @@ def pseudo_def(self) -> Pseudo | None: raise self.make_syntax_error("Expected {") if members := self.members(): if self.expect(lx.RBRACE) and self.expect(lx.SEMI): - return Pseudo( - tkn.text, members - ) + return Pseudo(tkn.text, members) return None def members(self) -> list[str] | None: diff --git a/Tools/cases_generator/plexer.py b/Tools/cases_generator/plexer.py index a73254ed5b1daa..cb6c5375866490 100644 --- a/Tools/cases_generator/plexer.py +++ b/Tools/cases_generator/plexer.py @@ -1,4 +1,5 @@ import lexer as lx + Token = lx.Token @@ -64,7 +65,9 @@ def require(self, kind: str) -> Token: tkn = self.next() if tkn is not None and tkn.kind == kind: return tkn - raise self.make_syntax_error(f"Expected {kind!r} but got {tkn and tkn.text!r}", tkn) + raise self.make_syntax_error( + f"Expected {kind!r} but got {tkn and tkn.text!r}", tkn + ) def extract_line(self, lineno: int) -> str: # Return source line `lineno` (1-based) @@ -73,18 +76,20 @@ def extract_line(self, lineno: int) -> str: return "" return lines[lineno - 1] - def make_syntax_error(self, message: str, tkn: Token|None = None) -> SyntaxError: + def make_syntax_error(self, message: str, tkn: Token | None = None) -> SyntaxError: # Construct a SyntaxError instance from message and token if tkn is None: tkn = self.peek() if tkn is None: tkn = self.tokens[-1] - return lx.make_syntax_error(message, - self.filename, tkn.line, tkn.column, self.extract_line(tkn.line)) + return lx.make_syntax_error( + message, self.filename, tkn.line, tkn.column, self.extract_line(tkn.line) + ) if __name__ == "__main__": import sys + if sys.argv[1:]: filename = sys.argv[1] if filename == "-c" and sys.argv[2:]: diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py new file mode 100644 index 00000000000000..23eca3037f896d --- /dev/null +++ b/Tools/cases_generator/stacking.py @@ -0,0 +1,400 @@ +import dataclasses +import typing + +from formatting import ( + Formatter, + UNUSED, + maybe_parenthesize, + parenthesize_cond, +) +from instructions import ( + ActiveCacheEffect, + Instruction, + MacroInstruction, + Component, + Tiers, + TIER_ONE, +) +from parsing import StackEffect, CacheEffect, Family + + +@dataclasses.dataclass +class StackOffset: + """Represent the stack offset for a PEEK or POKE. + + - At stack_pointer[0], deep and high are both empty. + (Note that that is an invalid stack reference.) + - Below stack top, only deep is non-empty. + - Above stack top, only high is non-empty. + - In complex cases, both deep and high may be non-empty. + + All this would be much simpler if all stack entries were the same + size, but with conditional and array effects, they aren't. + The offsets are each represented by a list of StackEffect objects. + The name in the StackEffects is unused. + """ + + deep: list[StackEffect] = dataclasses.field(default_factory=list) + high: list[StackEffect] = dataclasses.field(default_factory=list) + + def clone(self) -> "StackOffset": + return StackOffset(list(self.deep), list(self.high)) + + def negate(self) -> "StackOffset": + return StackOffset(list(self.high), list(self.deep)) + + def deeper(self, eff: StackEffect) -> None: + if eff in self.high: + self.high.remove(eff) + else: + self.deep.append(eff) + + def higher(self, eff: StackEffect) -> None: + if eff in self.deep: + self.deep.remove(eff) + else: + self.high.append(eff) + + def as_terms(self) -> list[tuple[str, str]]: + num = 0 + terms: list[tuple[str, str]] = [] + for eff in self.deep: + if eff.size: + terms.append(("-", maybe_parenthesize(eff.size))) + elif eff.cond and eff.cond != "1": + terms.append(("-", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) + elif eff.cond != "0": + num -= 1 + for eff in self.high: + if eff.size: + terms.append(("+", maybe_parenthesize(eff.size))) + elif eff.cond and eff.cond != "1": + terms.append(("+", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) + elif eff.cond != "0": + num += 1 + if num < 0: + terms.insert(0, ("-", str(-num))) + elif num > 0: + terms.append(("+", str(num))) + return terms + + def as_index(self) -> str: + terms = self.as_terms() + return make_index(terms) + + +def make_index(terms: list[tuple[str, str]]) -> str: + # Produce an index expression from the terms honoring PEP 8, + # surrounding binary ops with spaces but not unary minus + index = "" + for sign, term in terms: + if index: + index += f" {sign} {term}" + elif sign == "+": + index = term + else: + index = sign + term + return index or "0" + + +@dataclasses.dataclass +class StackItem: + offset: StackOffset + effect: StackEffect + + def as_variable(self, lax: bool = False) -> str: + """Return e.g. stack_pointer[-1].""" + terms = self.offset.as_terms() + if self.effect.size: + terms.insert(0, ("+", "stack_pointer")) + index = make_index(terms) + if self.effect.size: + res = index + else: + res = f"stack_pointer[{index}]" + if not lax: + # Check that we're not reading or writing above stack top. + # Skip this for output variable initialization (lax=True). + assert ( + self.effect in self.offset.deep and not self.offset.high + ), f"Push or pop above current stack level: {res}" + return res + + +@dataclasses.dataclass +class CopyEffect: + src: StackEffect + dst: StackEffect + + +class EffectManager: + """Manage stack effects and offsets for an instruction.""" + + instr: Instruction + active_caches: list[ActiveCacheEffect] + peeks: list[StackItem] + pokes: list[StackItem] + copies: list[CopyEffect] # See merge() + # Track offsets from stack pointer + min_offset: StackOffset + final_offset: StackOffset + + def __init__( + self, + instr: Instruction, + active_caches: list[ActiveCacheEffect], + pred: "EffectManager | None" = None, + ): + self.instr = instr + self.active_caches = active_caches + self.peeks = [] + self.pokes = [] + self.copies = [] + self.final_offset = pred.final_offset.clone() if pred else StackOffset() + for eff in reversed(instr.input_effects): + self.final_offset.deeper(eff) + self.peeks.append(StackItem(offset=self.final_offset.clone(), effect=eff)) + self.min_offset = self.final_offset.clone() + for eff in instr.output_effects: + self.pokes.append(StackItem(offset=self.final_offset.clone(), effect=eff)) + self.final_offset.higher(eff) + + if pred: + # Replace push(x) + pop(y) with copy(x, y). + # Check that the sources and destinations are disjoint. + sources: set[str] = set() + destinations: set[str] = set() + while ( + pred.pokes + and self.peeks + and pred.pokes[-1].effect == self.peeks[-1].effect + ): + src = pred.pokes.pop(-1).effect + dst = self.peeks.pop(0).effect + pred.final_offset.deeper(src) + if dst.name != UNUSED: + destinations.add(dst.name) + if dst.name != src.name: + sources.add(src.name) + self.copies.append(CopyEffect(src, dst)) + # TODO: Turn this into an error (pass an Analyzer instance?) + assert sources & destinations == set(), ( + pred.instr.name, + self.instr.name, + sources, + destinations, + ) + + def adjust_deeper(self, eff: StackEffect) -> None: + for peek in self.peeks: + peek.offset.deeper(eff) + for poke in self.pokes: + poke.offset.deeper(eff) + self.min_offset.deeper(eff) + self.final_offset.deeper(eff) + + def adjust_higher(self, eff: StackEffect) -> None: + for peek in self.peeks: + peek.offset.higher(eff) + for poke in self.pokes: + poke.offset.higher(eff) + self.min_offset.higher(eff) + self.final_offset.higher(eff) + + def adjust(self, offset: StackOffset) -> None: + for down in offset.deep: + self.adjust_deeper(down) + for up in offset.high: + self.adjust_higher(up) + + def adjust_inverse(self, offset: StackOffset) -> None: + for down in offset.deep: + self.adjust_higher(down) + for up in offset.high: + self.adjust_deeper(up) + + def collect_vars(self) -> dict[str, StackEffect]: + """Collect all variables, skipping unused ones.""" + vars: dict[str, StackEffect] = {} + + def add(eff: StackEffect) -> None: + if eff.name != UNUSED: + if eff.name in vars: + # TODO: Make this an error + assert vars[eff.name] == eff, ( + self.instr.name, + eff.name, + vars[eff.name], + eff, + ) + else: + vars[eff.name] = eff + + for copy in self.copies: + add(copy.src) + add(copy.dst) + for peek in self.peeks: + add(peek.effect) + for poke in self.pokes: + add(poke.effect) + + return vars + + +def less_than(a: StackOffset, b: StackOffset) -> bool: + # TODO: Handle more cases + if a.high != b.high: + return False + return a.deep[: len(b.deep)] == b.deep + + +def get_managers(parts: list[Component]) -> list[EffectManager]: + managers: list[EffectManager] = [] + pred: EffectManager | None = None + for part in parts: + mgr = EffectManager(part.instr, part.active_caches, pred) + managers.append(mgr) + pred = mgr + return managers + + +def get_stack_effect_info_for_macro(mac: MacroInstruction) -> tuple[str, str]: + """Get the stack effect info for a macro instruction. + + Returns a tuple (popped, pushed) where each is a string giving a + symbolic expression for the number of values popped/pushed. + """ + parts = [part for part in mac.parts if isinstance(part, Component)] + managers = get_managers(parts) + popped = StackOffset() + for mgr in managers: + if less_than(mgr.min_offset, popped): + popped = mgr.min_offset.clone() + # Compute pushed = final - popped + pushed = managers[-1].final_offset.clone() + for effect in popped.deep: + pushed.higher(effect) + for effect in popped.high: + pushed.deeper(effect) + return popped.negate().as_index(), pushed.as_index() + + +def write_single_instr( + instr: Instruction, out: Formatter, tier: Tiers = TIER_ONE +) -> None: + try: + write_components( + [Component(instr, instr.active_caches)], + out, + tier, + ) + except AssertionError as err: + raise AssertionError(f"Error writing instruction {instr.name}") from err + + +def write_macro_instr( + mac: MacroInstruction, out: Formatter, family: Family | None +) -> None: + parts = [part for part in mac.parts if isinstance(part, Component)] + + cache_adjust = 0 + for part in mac.parts: + match part: + case CacheEffect(size=size): + cache_adjust += size + case Component(instr=instr): + cache_adjust += instr.cache_offset + case _: + typing.assert_never(part) + + out.emit("") + with out.block(f"TARGET({mac.name})"): + if mac.predicted: + out.emit(f"PREDICTED({mac.name});") + out.static_assert_family_size(mac.name, family, cache_adjust) + try: + write_components(parts, out, TIER_ONE) + except AssertionError as err: + raise AssertionError(f"Error writing macro {mac.name}") from err + if cache_adjust: + out.emit(f"next_instr += {cache_adjust};") + out.emit("DISPATCH();") + + +def write_components( + parts: list[Component], + out: Formatter, + tier: Tiers, +) -> None: + managers = get_managers(parts) + + all_vars: dict[str, StackEffect] = {} + for mgr in managers: + for name, eff in mgr.collect_vars().items(): + if name in all_vars: + # TODO: Turn this into an error -- variable conflict + assert all_vars[name] == eff, ( + name, + mgr.instr.name, + all_vars[name], + eff, + ) + else: + all_vars[name] = eff + + # Declare all variables + for name, eff in all_vars.items(): + out.declare(eff, None) + + for mgr in managers: + if len(parts) > 1: + out.emit(f"// {mgr.instr.name}") + + for copy in mgr.copies: + if copy.src.name != copy.dst.name: + out.assign(copy.dst, copy.src) + for peek in mgr.peeks: + out.assign( + peek.effect, + StackEffect( + peek.as_variable(), + peek.effect.type, + peek.effect.cond, + peek.effect.size, + ), + ) + # Initialize array outputs + for poke in mgr.pokes: + if poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: + out.assign( + poke.effect, + StackEffect( + poke.as_variable(lax=True), + poke.effect.type, + poke.effect.cond, + poke.effect.size, + ), + ) + + if len(parts) == 1: + mgr.instr.write_body(out, 0, mgr.active_caches, tier) + else: + with out.block(""): + mgr.instr.write_body(out, -4, mgr.active_caches, tier) + + if mgr is managers[-1]: + out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) + # Use clone() since adjust_inverse() mutates final_offset + mgr.adjust_inverse(mgr.final_offset.clone()) + + for poke in mgr.pokes: + if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: + out.assign( + StackEffect( + poke.as_variable(), + poke.effect.type, + poke.effect.cond, + poke.effect.size, + ), + poke.effect, + ) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index cb999c156ee28b..423cdfb939c161 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -7,6 +7,7 @@ from __future__ import annotations import abc +import argparse import ast import builtins as bltns import collections @@ -27,7 +28,6 @@ import string import sys import textwrap -import traceback from collections.abc import ( Callable, @@ -37,13 +37,13 @@ ) from types import FunctionType, NoneType from typing import ( + TYPE_CHECKING, Any, Final, Literal, NamedTuple, NoReturn, Protocol, - TypeGuard, TypeVar, cast, overload, @@ -136,6 +136,28 @@ def text_accumulator() -> TextAccumulator: text, append, output = _text_accumulator() return TextAccumulator(append, output) + +@dc.dataclass +class ClinicError(Exception): + message: str + _: dc.KW_ONLY + lineno: int | None = None + filename: str | None = None + + def __post_init__(self) -> None: + super().__init__(self.message) + + def report(self, *, warn_only: bool = False) -> str: + msg = "Warning" if warn_only else "Error" + if self.filename is not None: + msg += f" in file {self.filename!r}" + if self.lineno is not None: + msg += f" on line {self.lineno}" + msg += ":\n" + msg += f"{self.message}\n" + return msg + + @overload def warn_or_fail( *args: object, @@ -159,25 +181,16 @@ def warn_or_fail( line_number: int | None = None, ) -> None: joined = " ".join([str(a) for a in args]) - add, output = text_accumulator() - if fail: - add("Error") - else: - add("Warning") if clinic: if filename is None: filename = clinic.filename if getattr(clinic, 'block_parser', None) and (line_number is None): line_number = clinic.block_parser.line_number - if filename is not None: - add(' in file "' + filename + '"') - if line_number is not None: - add(" on line " + str(line_number)) - add(':\n') - add(joined) - print(output()) + error = ClinicError(joined, filename=filename, lineno=line_number) if fail: - sys.exit(-1) + raise error + else: + print(error.report(warn_only=True)) def warn( @@ -277,9 +290,11 @@ def linear_format(s: str, **kwargs: str) -> str: continue if trailing: - fail("Text found after {" + name + "} block marker! It must be on a line by itself.") + fail(f"Text found after {{{name}}} block marker! " + "It must be on a line by itself.") if indent.strip(): - fail("Non-whitespace characters found before {" + name + "} block marker! It must be on a line by itself.") + fail(f"Non-whitespace characters found before {{{name}}} block marker! " + "It must be on a line by itself.") value = kwargs[name] if not value: @@ -346,7 +361,7 @@ def version_splitter(s: str) -> tuple[int, ...]: accumulator: list[str] = [] def flush() -> None: if not accumulator: - raise ValueError('Unsupported version string: ' + repr(s)) + fail(f'Unsupported version string: {s!r}') version.append(int(''.join(accumulator))) accumulator.clear() @@ -359,13 +374,15 @@ def flush() -> None: flush() version.append('abc'.index(c) - 3) else: - raise ValueError('Illegal character ' + repr(c) + ' in version string ' + repr(s)) + fail(f'Illegal character {c!r} in version string {s!r}') flush() return tuple(version) def version_comparitor(version1: str, version2: str) -> Literal[-1, 0, 1]: - iterator = itertools.zip_longest(version_splitter(version1), version_splitter(version2), fillvalue=0) - for i, (a, b) in enumerate(iterator): + iterator = itertools.zip_longest( + version_splitter(version1), version_splitter(version2), fillvalue=0 + ) + for a, b in iterator: if a < b: return -1 if a > b: @@ -759,6 +776,59 @@ class CLanguage(Language): stop_line = "[{dsl_name} start generated code]*/" checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" + PARSER_PROTOTYPE_KEYWORD: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) + """) + PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = normalize_snippet(""" + static int + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) + """) + PARSER_PROTOTYPE_VARARGS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args) + """) + PARSER_PROTOTYPE_FASTCALL: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) + """) + PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + """) + PARSER_PROTOTYPE_DEF_CLASS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + """) + PARSER_PROTOTYPE_NOARGS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) + """) + METH_O_PROTOTYPE: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({impl_parameters}) + """) + DOCSTRING_PROTOTYPE_VAR: Final[str] = normalize_snippet(""" + PyDoc_VAR({c_basename}__doc__); + """) + DOCSTRING_PROTOTYPE_STRVAR: Final[str] = normalize_snippet(""" + PyDoc_STRVAR({c_basename}__doc__, + {docstring}); + """) + IMPL_DEFINITION_PROTOTYPE: Final[str] = normalize_snippet(""" + static {impl_return_type} + {c_basename}_impl({impl_parameters}) + """) + METHODDEF_PROTOTYPE_DEFINE: Final[str] = normalize_snippet(r""" + #define {methoddef_name} \ + {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, + """) + METHODDEF_PROTOTYPE_IFNDEF: Final[str] = normalize_snippet(""" + #ifndef {methoddef_name} + #define {methoddef_name} + #endif /* !defined({methoddef_name}) */ + """) + def __init__(self, filename: str) -> None: super().__init__(filename) self.cpp = cpp.Monitor(filename) @@ -784,9 +854,6 @@ def docstring_for_c_string( self, f: Function ) -> str: - if re.search(r'[^\x00-\x7F]', f.docstring): - warn("Non-ascii character appear in docstring.") - text, add, output = _text_accumulator() # turn docstring into a properly quoted C string for line in f.docstring.split('\n'): @@ -805,7 +872,8 @@ def docstring_for_c_string( def output_templates( self, - f: Function + f: Function, + clinic: Clinic ) -> dict[str, str]: parameters = list(f.parameters.values()) assert parameters @@ -818,9 +886,7 @@ def output_templates( converters = [p.converter for p in parameters] has_option_groups = parameters and (parameters[0].group or parameters[-1].group) - default_return_converter = (not f.return_converter or - f.return_converter.type == 'PyObject *') - + default_return_converter = f.return_converter.type == 'PyObject *' new_or_init = f.kind.new_or_init vararg: int | str = NO_VARARG @@ -863,52 +929,15 @@ def output_templates( # methoddef_ifndef return_value_declaration = "PyObject *return_value = NULL;" - - methoddef_define = normalize_snippet(""" - #define {methoddef_name} \\ - {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, - """) + methoddef_define = self.METHODDEF_PROTOTYPE_DEFINE if new_or_init and not f.docstring: docstring_prototype = docstring_definition = '' else: - docstring_prototype = normalize_snippet(""" - PyDoc_VAR({c_basename}__doc__); - """) - docstring_definition = normalize_snippet(""" - PyDoc_STRVAR({c_basename}__doc__, - {docstring}); - """) - impl_definition = normalize_snippet(""" - static {impl_return_type} - {c_basename}_impl({impl_parameters}) - """) + docstring_prototype = self.DOCSTRING_PROTOTYPE_VAR + docstring_definition = self.DOCSTRING_PROTOTYPE_STRVAR + impl_definition = self.IMPL_DEFINITION_PROTOTYPE impl_prototype = parser_prototype = parser_definition = None - parser_prototype_keyword = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) - - parser_prototype_varargs = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args) - """) - - parser_prototype_fastcall = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) - """) - - parser_prototype_fastcall_keywords = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - - parser_prototype_def_class = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - # parser_body_fields remembers the fields passed in to the # previous call to parser_body. this is used for an awful hack. parser_body_fields: tuple[str, ...] = () @@ -951,19 +980,13 @@ def parser_body( if not requires_defining_class: # no parameters, METH_NOARGS flags = "METH_NOARGS" - - parser_prototype = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) - """) + parser_prototype = self.PARSER_PROTOTYPE_NOARGS parser_code = [] - else: assert not new_or_init flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" - - parser_prototype = parser_prototype_def_class + parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS return_error = ('return NULL;' if default_return_converter else 'goto exit;') parser_code = [normalize_snippet(""" @@ -988,10 +1011,7 @@ def parser_body( if (isinstance(converters[0], object_converter) and converters[0].format_unit == 'O'): - meth_o_prototype = normalize_snippet(""" - static PyObject * - {c_basename}({impl_parameters}) - """) + meth_o_prototype = self.METH_O_PROTOTYPE if default_return_converter: # maps perfectly to METH_O, doesn't need a return converter. @@ -1031,8 +1051,7 @@ def parser_body( # in a big switch statement) flags = "METH_VARARGS" - parser_prototype = parser_prototype_varargs - + parser_prototype = self.PARSER_PROTOTYPE_VARARGS parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: @@ -1041,7 +1060,7 @@ def parser_body( # we only need one call to _PyArg_ParseStack flags = "METH_FASTCALL" - parser_prototype = parser_prototype_fastcall + parser_prototype = self.PARSER_PROTOTYPE_FASTCALL nargs = 'nargs' argname_fmt = 'args[%d]' else: @@ -1049,7 +1068,7 @@ def parser_body( # we only need one call to PyArg_ParseTuple flags = "METH_VARARGS" - parser_prototype = parser_prototype_varargs + parser_prototype = self.PARSER_PROTOTYPE_VARARGS nargs = 'PyTuple_GET_SIZE(args)' argname_fmt = 'PyTuple_GET_ITEM(args, %d)' @@ -1096,7 +1115,6 @@ def parser_body( parsearg = p.converter.parse_arg(argname, displayname) if parsearg is None: - #print('Cannot convert %s %r for %s' % (p.converter.__class__.__name__, p.converter.format_unit, p.converter.name), file=sys.stderr) parser_code = None break if has_optional or p.is_optional(): @@ -1147,7 +1165,7 @@ def parser_body( nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" if not new_or_init: flags = "METH_FASTCALL|METH_KEYWORDS" - parser_prototype = parser_prototype_fastcall_keywords + parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS argname_fmt = 'args[%d]' declarations = declare_parser(f) declarations += "\nPyObject *argsbuf[%s];" % len(converters) @@ -1162,7 +1180,7 @@ def parser_body( else: # positional-or-keyword arguments flags = "METH_VARARGS|METH_KEYWORDS" - parser_prototype = parser_prototype_keyword + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD argname_fmt = 'fastargs[%d]' declarations = declare_parser(f) declarations += "\nPyObject *argsbuf[%s];" % len(converters) @@ -1179,7 +1197,7 @@ def parser_body( if requires_defining_class: flags = 'METH_METHOD|' + flags - parser_prototype = parser_prototype_def_class + parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS add_label: str | None = None for i, p in enumerate(parameters): @@ -1189,7 +1207,6 @@ def parser_body( displayname = p.get_displayname(i+1) parsearg = p.converter.parse_arg(argname_fmt % i, displayname) if parsearg is None: - #print('Cannot convert %s %r for %s' % (p.converter.__class__.__name__, p.converter.format_unit, p.converter.name), file=sys.stderr) parser_code = None break if add_label and (i == pos_only or i == max_pos): @@ -1267,13 +1284,10 @@ def parser_body( methoddef_define = '' if f.kind is METHOD_NEW: - parser_prototype = parser_prototype_keyword + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD else: return_value_declaration = "int return_value = -1;" - parser_prototype = normalize_snippet(""" - static int - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD___INIT__ fields = list(parser_body_fields) parses_positional = 'METH_NOARGS' not in flags @@ -1324,15 +1338,9 @@ def parser_body( cpp_if = "#if " + conditional cpp_endif = "#endif /* " + conditional + " */" - assert clinic is not None if methoddef_define and f.full_name not in clinic.ifndef_symbols: clinic.ifndef_symbols.add(f.full_name) - methoddef_ifndef = normalize_snippet(""" - #ifndef {methoddef_name} - #define {methoddef_name} - #endif /* !defined({methoddef_name}) */ - """) - + methoddef_ifndef = self.METHODDEF_PROTOTYPE_IFNDEF # add ';' to the end of parser_prototype and impl_prototype # (they mustn't be None, but they could be an empty string.) @@ -1490,7 +1498,7 @@ def render_function( parameters = f.render_parameters converters = [p.converter for p in parameters] - templates = self.output_templates(f) + templates = self.output_templates(f, clinic) f_self = parameters[0] selfless = parameters[1:] @@ -1526,7 +1534,8 @@ def render_function( c.render(p, data) if has_option_groups and (not positional): - fail("You cannot use optional groups ('[' and ']')\nunless all parameters are positional-only ('/').") + fail("You cannot use optional groups ('[' and ']') " + "unless all parameters are positional-only ('/').") # HACK # when we're METH_O, but have a custom return converter, @@ -1675,27 +1684,30 @@ class Block: found on the start line of the block between the square brackets. - signatures is either list or None. If it's a list, - it may only contain clinic.Module, clinic.Class, and + signatures is a list. + It may only contain clinic.Module, clinic.Class, and clinic.Function objects. At the moment it should contain at most one of each. output is either str or None. If str, it's the output from this block, with embedded '\n' characters. - indent is either str or None. It's the leading whitespace + indent is a str. It's the leading whitespace that was found on every line of input. (If body_prefix is not empty, this is the indent *after* removing the body_prefix.) - preindent is either str or None. It's the whitespace that + "indent" is different from the concept of "preindent" + (which is not stored as state on Block objects). + "preindent" is the whitespace that was found in front of every line of input *before* the "body_prefix" (see the Language object). If body_prefix is empty, preindent must always be empty too. - To illustrate indent and preindent: Assume that '_' - represents whitespace. If the block processed was in a - Python file, and looked like this: + To illustrate the difference between "indent" and "preindent": + + Assume that '_' represents whitespace. + If the block processed was in a Python file, and looked like this: ____#/*[python] ____#__for a in range(20): ____#____print(a) @@ -1708,7 +1720,6 @@ class Block: signatures: list[Module | Class | Function] = dc.field(default_factory=list) output: Any = None # TODO: Very dynamic; probably untypeable in its current form? indent: str = '' - preindent: str = '' def __repr__(self) -> str: dsl_name = self.dsl_name or "text" @@ -1717,8 +1728,12 @@ def summarize(s: object) -> str: if len(s) > 30: return s[:26] + "..." + s[0] return s - return "".join(( - "")) + parts = ( + repr(dsl_name), + f"input={summarize(self.input)}", + f"output={summarize(self.output)}" + ) + return f"" class BlockParser: @@ -1863,7 +1878,7 @@ def is_stop_line(line: str) -> bool: for field in shlex.split(arguments): name, equals, value = field.partition('=') if not equals: - fail("Mangled Argument Clinic marker line:", repr(line)) + fail(f"Mangled Argument Clinic marker line: {line!r}") d[name.strip()] = value.strip() if self.verify: @@ -1874,11 +1889,10 @@ def is_stop_line(line: str) -> bool: computed = compute_checksum(output, len(checksum)) if checksum != computed: - fail("Checksum mismatch!\nExpected: {}\nComputed: {}\n" + fail("Checksum mismatch! " + f"Expected {checksum!r}, computed {computed!r}. " "Suggested fix: remove all generated code including " - "the end marker,\n" - "or use the '-f' option." - .format(checksum, computed)) + "the end marker, or use the '-f' option.") else: # put back output output_lines = output.splitlines(keepends=True) @@ -2009,13 +2023,13 @@ def __post_init__(self, args: tuple[str, ...]) -> None: ) extra_arguments = 1 if self.type == "file" else 0 if len(args) < extra_arguments: - fail(f"Not enough arguments for destination {self.name} new {self.type}") + fail(f"Not enough arguments for destination " + f"{self.name!r} new {self.type!r}") if len(args) > extra_arguments: - fail(f"Too many arguments for destination {self.name} new {self.type}") + fail(f"Too many arguments for destination {self.name!r} new {self.type!r}") if self.type =='file': d = {} filename = self.clinic.filename - assert filename is not None d['path'] = filename dirname, basename = os.path.split(filename) if not dirname: @@ -2027,26 +2041,22 @@ def __post_init__(self, args: tuple[str, ...]) -> None: def __repr__(self) -> str: if self.type == 'file': - file_repr = " " + repr(self.filename) + type_repr = f"type='file' file={self.filename!r}" else: - file_repr = '' - return "".join(("")) + type_repr = f"type={self.type!r}" + return f"" def clear(self) -> None: if self.type != 'buffer': - fail("Can't clear destination" + self.name + " , it's not of type buffer") + fail(f"Can't clear destination {self.name!r}: it's not of type 'buffer'") self.buffers.clear() def dump(self) -> str: return self.buffers.dump() -# maps strings to Language objects. -# "languages" maps the name of the language ("C", "Python"). -# "extensions" maps the file extension ("c", "py"). +# "extensions" maps the file extension ("c", "py") to Language classes. LangDict = dict[str, Callable[[str], Language]] - -languages = { 'C': CLanguage, 'Python': PythonLanguage } extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } extensions['py'] = PythonLanguage @@ -2133,8 +2143,8 @@ def __init__( language: CLanguage, printer: BlockPrinter | None = None, *, + filename: str, verify: bool = True, - filename: str | None = None ) -> None: # maps strings to Parser objects. # (instantiated from the "parsers" global.) @@ -2213,13 +2223,13 @@ def add_destination( *args: str ) -> None: if name in self.destinations: - fail("Destination already exists: " + repr(name)) + fail(f"Destination already exists: {name!r}") self.destinations[name] = Destination(name, type, self, args) def get_destination(self, name: str) -> Destination: d = self.destinations.get(name) if not d: - fail("Destination does not exist: " + repr(name)) + fail(f"Destination does not exist: {name!r}") return d def get_destination_buffer( @@ -2240,11 +2250,7 @@ def parse(self, input: str) -> str: assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block." self.parsers[dsl_name] = parsers[dsl_name](self) parser = self.parsers[dsl_name] - try: - parser.parse(block) - except Exception: - fail('Exception raised during parsing:\n' + - traceback.format_exc().rstrip()) + parser.parse(block) printer.print_block(block) # these are destinations not buffers @@ -2270,15 +2276,16 @@ def parse(self, input: str) -> str: os.makedirs(dirname) except FileExistsError: if not os.path.isdir(dirname): - fail("Can't write to destination {}, " - "can't make directory {}!".format( - destination.filename, dirname)) + fail(f"Can't write to destination " + f"{destination.filename!r}; " + f"can't make directory {dirname!r}!") if self.verify: with open(destination.filename) as f: parser_2 = BlockParser(f.read(), language=self.language) blocks = list(parser_2) if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'): - fail("Modified destination file " + repr(destination.filename) + ", not overwriting!") + fail(f"Modified destination file " + f"{destination.filename!r}; not overwriting!") except FileNotFoundError: pass @@ -2319,7 +2326,8 @@ def _module_and_class( return module, cls child = parent.classes.get(field) if not child: - fail('Parent class or module ' + '.'.join(so_far) + " does not exist.") + fullname = ".".join(so_far) + fail(f"Parent class or module {fullname!r} does not exist.") cls = parent = child return module, cls @@ -2336,12 +2344,12 @@ def parse_file( extension = os.path.splitext(filename)[1][1:] if not extension: - fail("Can't extract file type for file " + repr(filename)) + fail(f"Can't extract file type for file {filename!r}") try: language = extensions[extension](filename) except KeyError: - fail("Can't identify file type for file " + repr(filename)) + fail(f"Can't identify file type for file {filename!r}") with open(filename, encoding="utf-8") as f: raw = f.read() @@ -2496,7 +2504,7 @@ def new_or_init(self) -> bool: return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} def __repr__(self) -> str: - return f"" + return f"" INVALID: Final = FunctionKind.INVALID @@ -2573,7 +2581,7 @@ def methoddef_flags(self) -> str | None: return '|'.join(flags) def __repr__(self) -> str: - return '' + return f'' def copy(self, **overrides: Any) -> Function: f = dc.replace(self, **overrides) @@ -2601,7 +2609,7 @@ class Parameter: right_bracket_count: int = dc.field(init=False, default=0) def __repr__(self) -> str: - return '' + return f'' def is_keyword_only(self) -> bool: return self.kind == inspect.Parameter.KEYWORD_ONLY @@ -2638,18 +2646,6 @@ def get_displayname(self, i: int) -> str: return f'"argument {i}"' -@dc.dataclass -class LandMine: - # try to access any - __message__: str - - def __getattribute__(self, name: str) -> Any: - if name in ('__repr__', '__message__'): - return super().__getattribute__(name) - # raise RuntimeError(repr(name)) - fail("Stepped on a land mine, trying to access attribute " + repr(name) + ":\n" + self.__message__) - - CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) def add_c_converter( @@ -2787,7 +2783,7 @@ class CConverter(metaclass=CConverterAutoRegister): # Only used by the 'O!' format unit (and the "object" converter). subclass_of: str | None = None - # Do we want an adjacent '_length' variable for this variable? + # See also the 'length_name' property. # Only used by format units ending with '#'. length = False @@ -2832,8 +2828,9 @@ def __init__(self, else: names = [cls.__name__ for cls in self.default_type] types_str = ', '.join(names) - fail("{}: default value {!r} for field {} is not of type {}".format( - self.__class__.__name__, default, name, types_str)) + cls_name = self.__class__.__name__ + fail(f"{cls_name}: default value {default!r} for field " + f"{name!r} is not of type {types_str!r}") self.default = default if c_default: @@ -2844,16 +2841,28 @@ def __init__(self, if annotation is not unspecified: fail("The 'annotation' parameter is not currently permitted.") - # this is deliberate, to prevent you from caching information - # about the function in the init. - # (that breaks if we get cloned.) - # so after this change we will noisily fail. - self.function: Function | LandMine = LandMine( - "Don't access members of self.function inside converter_init!" - ) + # Make sure not to set self.function until after converter_init() has been called. + # This prevents you from caching information + # about the function in converter_init(). + # (That breaks if we get cloned.) self.converter_init(**kwargs) self.function = function + # Add a custom __getattr__ method to improve the error message + # if somebody tries to access self.function in converter_init(). + # + # mypy will assume arbitrary access is okay for a class with a __getattr__ method, + # and that's not what we want, + # so put it inside an `if not TYPE_CHECKING` block + if not TYPE_CHECKING: + def __getattr__(self, attr): + if attr == "function": + fail( + f"{self.__class__.__name__!r} object has no attribute 'function'.\n" + f"Note: accessing self.function inside converter_init is disallowed!" + ) + return super().__getattr__(attr) + def converter_init(self) -> None: pass @@ -2868,12 +2877,12 @@ def _render_self(self, parameter: Parameter, data: CRenderData) -> None: s = ("&" if self.impl_by_reference else "") + name data.impl_arguments.append(s) if self.length: - data.impl_arguments.append(self.length_name()) + data.impl_arguments.append(self.length_name) # impl_parameters data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) if self.length: - data.impl_parameters.append("Py_ssize_t " + self.length_name()) + data.impl_parameters.append(f"Py_ssize_t {self.length_name}") def _render_non_self( self, @@ -2932,6 +2941,7 @@ def render(self, parameter: Parameter, data: CRenderData) -> None: self._render_self(parameter, data) self._render_non_self(parameter, data) + @functools.cached_property def length_name(self) -> str: """Computes the name of the associated "length" variable.""" assert self.length is not None @@ -2955,7 +2965,7 @@ def parse_argument(self, args: list[str]) -> None: args.append(s) if self.length: - args.append("&" + self.length_name()) + args.append(f"&{self.length_name}") # # All the functions after here are intended as extension points. @@ -3000,9 +3010,8 @@ def declaration(self, *, in_parser: bool = False) -> str: declaration.append(default) declaration.append(";") if self.length: - declaration.append('\nPy_ssize_t ') - declaration.append(self.length_name()) - declaration.append(';') + declaration.append('\n') + declaration.append(f"Py_ssize_t {self.length_name};") return "".join(declaration) def initialize(self) -> str: @@ -3143,7 +3152,7 @@ def converter_init(self, *, accept: TypeSet = {object}) -> None: if accept == {int}: self.format_unit = 'i' elif accept != {object}: - fail("bool_converter: illegal 'accept' argument " + repr(accept)) + fail(f"bool_converter: illegal 'accept' argument {accept!r}") if self.default is not unspecified: self.default = bool(self.default) self.c_default = str(int(self.default)) @@ -3193,7 +3202,7 @@ class char_converter(CConverter): def converter_init(self) -> None: if isinstance(self.default, self.default_type): if len(self.default) != 1: - fail("char_converter: illegal default value " + repr(self.default)) + fail(f"char_converter: illegal default value {self.default!r}") self.c_default = repr(bytes(self.default))[1:] if self.c_default == '"\'"': @@ -3332,7 +3341,7 @@ def converter_init( if accept == {str}: self.format_unit = 'C' elif accept != {int}: - fail("int_converter: illegal 'accept' argument " + repr(accept)) + fail(f"int_converter: illegal 'accept' argument {accept!r}") if type is not None: self.type = type @@ -3469,7 +3478,7 @@ def converter_init(self, *, accept: TypeSet = {int}) -> None: elif accept == {int, NoneType}: self.converter = '_Py_convert_optional_to_ssize_t' else: - fail("Py_ssize_t_converter: illegal 'accept' argument " + repr(accept)) + fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") def parse_arg(self, argname: str, displayname: str) -> str | None: if self.format_unit == 'n': @@ -3499,7 +3508,7 @@ def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: elif accept == {int, NoneType}: self.converter = '_PyEval_SliceIndex' else: - fail("slice_index_converter: illegal 'accept' argument " + repr(accept)) + fail(f"slice_index_converter: illegal 'accept' argument {accept!r}") class size_t_converter(CConverter): type = 'size_t' @@ -3681,29 +3690,29 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: _PyArg_BadArgument("{{name}}", {displayname}, "str", {argname}); goto exit; }}}} - Py_ssize_t {paramname}_length; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{paramname}_length); + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); if ({paramname} == NULL) {{{{ goto exit; }}}} - if (strlen({paramname}) != (size_t){paramname}_length) {{{{ + if (strlen({paramname}) != (size_t){length_name}) {{{{ PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; }}}} """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) + displayname=displayname, length_name=self.length_name) if self.format_unit == 'z': return """ if ({argname} == Py_None) {{{{ {paramname} = NULL; }}}} else if (PyUnicode_Check({argname})) {{{{ - Py_ssize_t {paramname}_length; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{paramname}_length); + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); if ({paramname} == NULL) {{{{ goto exit; }}}} - if (strlen({paramname}) != (size_t){paramname}_length) {{{{ + if (strlen({paramname}) != (size_t){length_name}) {{{{ PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; }}}} @@ -3713,7 +3722,7 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: goto exit; }}}} """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) + displayname=displayname, length_name=self.length_name) return super().parse_arg(argname, displayname) # @@ -3847,7 +3856,7 @@ def converter_init( elif accept == {str, NoneType}: self.converter = '_PyUnicode_WideCharString_Opt_Converter' else: - fail("Py_UNICODE_converter: illegal 'accept' argument " + repr(accept)) + fail(f"Py_UNICODE_converter: illegal 'accept' argument {accept!r}") self.c_default = "NULL" def cleanup(self) -> str: @@ -4005,7 +4014,6 @@ def converter_init(self, *, type: str | None = None) -> None: def pre_render(self) -> None: f = self.function - assert isinstance(f, Function) default_type, default_name = correct_name_for_self(f) self.signature_name = default_name self.type = self.specified_type or self.type or default_type @@ -4056,7 +4064,6 @@ def pre_render(self) -> None: @property def parser_type(self) -> str: assert self.type is not None - assert isinstance(self.function, Function) return required_type_for_self_for_parser(self.function) or self.type def render(self, parameter: Parameter, data: CRenderData) -> None: @@ -4369,27 +4376,19 @@ def depth(self) -> int: """ return len(self.indents) - def indent(self, line: str) -> str: - """ - Indents a line by the currently defined margin. - """ - assert self.margin is not None, "Cannot call .indent() before calling .infer()" - return self.margin + line - def dedent(self, line: str) -> str: """ Dedents a line by the currently defined margin. - (The inverse of 'indent'.) """ - assert self.margin is not None, "Cannot call .indent() before calling .infer()" + assert self.margin is not None, "Cannot call .dedent() before calling .infer()" margin = self.margin indent = self.indents[-1] if not line.startswith(margin): - fail('Cannot dedent, line does not start with the previous margin:') + fail('Cannot dedent; line does not start with the previous margin.') return line[indent:] -StateKeeper = Callable[[str | None], None] +StateKeeper = Callable[[str], None] ConverterArgs = dict[str, Any] class ParamState(enum.IntEnum): @@ -4429,8 +4428,7 @@ class DSLParser: keyword_only: bool positional_only: bool group: int - parameter_state: int - seen_positional_with_default: bool + parameter_state: ParamState indent: IndentStack kind: FunctionKind coexist: bool @@ -4461,7 +4459,6 @@ def reset(self) -> None: self.positional_only = False self.group = 0 self.parameter_state: ParamState = ParamState.START - self.seen_positional_with_default = False self.indent = IndentStack() self.kind = CALLABLE self.coexist = False @@ -4471,7 +4468,9 @@ def reset(self) -> None: def directive_version(self, required: str) -> None: global version if version_comparitor(version, required) < 0: - fail("Insufficient Clinic version!\n Version: " + version + "\n Required: " + required) + fail("Insufficient Clinic version!\n" + f" Version: {version}\n" + f" Required: {required}") def directive_module(self, name: str) -> None: fields = name.split('.')[:-1] @@ -4479,8 +4478,8 @@ def directive_module(self, name: str) -> None: if cls: fail("Can't nest a module inside a class!") - if name in module.classes: - fail("Already defined module " + repr(name) + "!") + if name in module.modules: + fail(f"Already defined module {name!r}!") m = Module(name, module) module.modules[name] = m @@ -4498,7 +4497,7 @@ def directive_class( parent = cls or module if name in parent.classes: - fail("Already defined class " + repr(name) + "!") + fail(f"Already defined class {name!r}!") c = Class(name, module, cls, typedef, type_object) parent.classes[name] = c @@ -4506,7 +4505,7 @@ def directive_class( def directive_set(self, name: str, value: str) -> None: if name not in ("line_prefix", "line_suffix"): - fail("unknown variable", repr(name)) + fail(f"unknown variable {name!r}") value = value.format_map({ 'block comment start': '/*', @@ -4527,7 +4526,7 @@ def directive_destination( case "clear": self.clinic.get_destination(name).clear() case _: - fail("unknown destination command", repr(command)) + fail(f"unknown destination command {command!r}") def directive_output( @@ -4540,7 +4539,7 @@ def directive_output( if command_or_name == "preset": preset = self.clinic.presets.get(destination) if not preset: - fail("Unknown preset " + repr(destination) + "!") + fail(f"Unknown preset {destination!r}!") fd.update(preset) return @@ -4569,7 +4568,11 @@ def directive_output( return if command_or_name not in fd: - fail("Invalid command / destination name " + repr(command_or_name) + ", must be one of:\n preset push pop print everything " + " ".join(fd)) + allowed = ["preset", "push", "pop", "print", "everything"] + allowed.extend(fd) + fail(f"Invalid command or destination name {command_or_name!r}. " + "Must be one of:\n -", + "\n - ".join([repr(word) for word in allowed])) fd[command_or_name] = d def directive_dump(self, name: str) -> None: @@ -4581,7 +4584,7 @@ def directive_printout(self, *args: str) -> None: def directive_preserve(self) -> None: if self.preserve_output: - fail("Can't have preserve twice in one block!") + fail("Can't have 'preserve' twice in one block!") self.preserve_output = True def at_classmethod(self) -> None: @@ -4608,51 +4611,51 @@ def parse(self, block: Block) -> None: lines = block.input.split('\n') for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): if '\t' in line: - fail('Tab characters are illegal in the Clinic DSL.\n\t' + repr(line), line_number=block_start) - self.state(line) - - self.next(self.state_terminal) - self.state(None) + fail(f'Tab characters are illegal in the Clinic DSL: {line!r}', + line_number=block_start) + try: + self.state(line) + except ClinicError as exc: + exc.lineno = line_number + raise - block.output.extend(self.clinic.language.render(clinic, block.signatures)) + self.do_post_block_processing_cleanup() + block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) if self.preserve_output: if block.output: fail("'preserve' only works for blocks that don't produce any output!") block.output = self.saved_output - @staticmethod - def valid_line(line: str | None) -> TypeGuard[str]: - if line is None: - return False + def in_docstring(self) -> bool: + """Return true if we are processing a docstring.""" + return self.state in { + self.state_parameter_docstring, + self.state_function_docstring, + } + def valid_line(self, line: str) -> bool: # ignore comment-only lines if line.lstrip().startswith('#'): return False # Ignore empty lines too # (but not in docstring sections!) - if not line.strip(): + if not self.in_docstring() and not line.strip(): return False return True - @staticmethod - def calculate_indent(line: str) -> int: - return len(line) - len(line.strip()) - def next( self, state: StateKeeper, line: str | None = None ) -> None: - # real_print(self.state.__name__, "->", state.__name__, ", line=", line) self.state = state if line is not None: self.state(line) - def state_dsl_start(self, line: str | None) -> None: - # self.block = self.ClinicOutputBlock(self) + def state_dsl_start(self, line: str) -> None: if not self.valid_line(line): return @@ -4669,7 +4672,7 @@ def state_dsl_start(self, line: str | None) -> None: self.next(self.state_modulename_name, line) - def state_modulename_name(self, line: str | None) -> None: + def state_modulename_name(self, line: str) -> None: # looking for declaration, which establishes the leftmost column # line should be # modulename.fnname [as c_basename] [-> return annotation] @@ -4686,9 +4689,7 @@ def state_modulename_name(self, line: str | None) -> None: # this line is permitted to start with whitespace. # we'll call this number of spaces F (for "function"). - if not self.valid_line(line): - return - + assert self.valid_line(line) self.indent.infer(line) # are we cloning? @@ -4711,8 +4712,8 @@ def state_modulename_name(self, line: str | None) -> None: if existing_function.name == function_name: break else: - print(f"{cls=}, {module=}, {existing=}") - print(f"{(cls or module).functions=}") + print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) + print(f"{(cls or module).functions=}", file=sys.stderr) fail(f"Couldn't find existing function {existing!r}!") fields = [x.strip() for x in full_name.split('.')] @@ -4720,7 +4721,8 @@ def state_modulename_name(self, line: str | None) -> None: module, cls = self.clinic._module_and_class(fields) if not (existing_function.kind is self.kind and existing_function.coexist == self.coexist): - fail("'kind' of function and cloned function don't match! (@classmethod/@staticmethod/@coexist)") + fail("'kind' of function and cloned function don't match! " + "(@classmethod/@staticmethod/@coexist)") function = existing_function.copy( name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, docstring='' @@ -4739,9 +4741,9 @@ def state_modulename_name(self, line: str | None) -> None: c_basename = c_basename.strip() or None if not is_legal_py_identifier(full_name): - fail("Illegal function name:", full_name) + fail(f"Illegal function name: {full_name!r}") if c_basename and not is_legal_c_identifier(c_basename): - fail("Illegal C basename:", c_basename) + fail(f"Illegal C basename: {c_basename!r}") return_converter = None if returns: @@ -4749,7 +4751,7 @@ def state_modulename_name(self, line: str | None) -> None: try: module_node = ast.parse(ast_input) except SyntaxError: - fail(f"Badly formed annotation for {full_name}: {returns!r}") + fail(f"Badly formed annotation for {full_name!r}: {returns!r}") function_node = module_node.body[0] assert isinstance(function_node, ast.FunctionDef) try: @@ -4760,7 +4762,7 @@ def state_modulename_name(self, line: str | None) -> None: fail(f"No available return converter called {name!r}") return_converter = return_converters[name](**kwargs) except ValueError: - fail(f"Badly formed annotation for {full_name}: {returns!r}") + fail(f"Badly formed annotation for {full_name!r}: {returns!r}") fields = [x.strip() for x in full_name.split('.')] function_name = fields.pop() @@ -4784,8 +4786,6 @@ def state_modulename_name(self, line: str | None) -> None: if not return_converter: return_converter = CReturnConverter() - if not module: - fail("Undefined module used in declaration of " + repr(full_name.strip()) + ".") self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, return_converter=return_converter, kind=self.kind, coexist=self.coexist) self.block.signatures.append(self.function) @@ -4858,7 +4858,7 @@ def state_modulename_name(self, line: str | None) -> None: # separate boolean state variables.) The states are defined in the # ParamState class. - def state_parameters_start(self, line: str | None) -> None: + def state_parameters_start(self, line: str) -> None: if not self.valid_line(line): return @@ -4880,7 +4880,7 @@ def to_required(self) -> None: for p in self.function.parameters.values(): p.group = -p.group - def state_parameter(self, line: str | None) -> None: + def state_parameter(self, line: str) -> None: assert isinstance(self.function, Function) if not self.valid_line(line): @@ -4971,18 +4971,22 @@ def parse_parameter(self, line: str) -> None: except SyntaxError: pass if not module: - fail("Function " + self.function.name + " has an invalid parameter declaration:\n\t" + line) + fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", + repr(line)) function = module.body[0] assert isinstance(function, ast.FunctionDef) function_args = function.args if len(function_args.args) > 1: - fail("Function " + self.function.name + " has an invalid parameter declaration (comma?):\n\t" + line) + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (comma?): {line!r}") if function_args.defaults or function_args.kw_defaults: - fail("Function " + self.function.name + " has an invalid parameter declaration (default value?):\n\t" + line) + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (default value?): {line!r}") if function_args.kwarg: - fail("Function " + self.function.name + " has an invalid parameter declaration (**kwargs?):\n\t" + line) + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (**kwargs?): {line!r}") if function_args.vararg: is_vararg = True @@ -4996,7 +5000,7 @@ def parse_parameter(self, line: str) -> None: if not default: if self.parameter_state is ParamState.OPTIONAL: - fail(f"Can't have a parameter without a default ({parameter_name!r})\n" + fail(f"Can't have a parameter without a default ({parameter_name!r}) " "after a parameter with a default!") value: Sentinels | Null if is_vararg: @@ -5056,10 +5060,10 @@ def bad_node(self, node: ast.AST) -> None: except NameError: pass # probably a named constant except Exception as e: - fail("Malformed expression given as default value\n" - "{!r} caused {!r}".format(default, e)) + fail("Malformed expression given as default value " + f"{default!r} caused {e!r}") if bad: - fail("Unsupported expression as default value: " + repr(default)) + fail(f"Unsupported expression as default value: {default!r}") assignment = module.body[0] assert isinstance(assignment, ast.Assign) @@ -5077,7 +5081,10 @@ def bad_node(self, node: ast.AST) -> None: )): c_default = kwargs.get("c_default") if not (isinstance(c_default, str) and c_default): - fail("When you specify an expression (" + repr(default) + ") as your default value,\nyou MUST specify a valid c_default." + ast.dump(expr)) + fail(f"When you specify an expression ({default!r}) " + f"as your default value, " + f"you MUST specify a valid c_default.", + ast.dump(expr)) py_default = default value = unknown elif isinstance(expr, ast.Attribute): @@ -5087,13 +5094,16 @@ def bad_node(self, node: ast.AST) -> None: a.append(n.attr) n = n.value if not isinstance(n, ast.Name): - fail("Unsupported default value " + repr(default) + " (looked like a Python constant)") + fail(f"Unsupported default value {default!r} " + "(looked like a Python constant)") a.append(n.id) py_default = ".".join(reversed(a)) c_default = kwargs.get("c_default") if not (isinstance(c_default, str) and c_default): - fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") + fail(f"When you specify a named constant ({py_default!r}) " + "as your default value, " + "you MUST specify a valid c_default.") try: value = eval(py_default) @@ -5110,13 +5120,15 @@ def bad_node(self, node: ast.AST) -> None: c_default = py_default except SyntaxError as e: - fail("Syntax error: " + repr(e.text)) + fail(f"Syntax error: {e.text!r}") except (ValueError, AttributeError): value = unknown c_default = kwargs.get("c_default") py_default = default if not (isinstance(c_default, str) and c_default): - fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") + fail("When you specify a named constant " + f"({py_default!r}) as your default value, " + "you MUST specify a valid c_default.") kwargs.setdefault('c_default', c_default) kwargs.setdefault('py_default', py_default) @@ -5124,7 +5136,7 @@ def bad_node(self, node: ast.AST) -> None: dict = legacy_converters if legacy else converters legacy_str = "legacy " if legacy else "" if name not in dict: - fail(f'{name} is not a valid {legacy_str}converter') + fail(f'{name!r} is not a valid {legacy_str}converter') # if you use a c_name for the parameter, we just give that name to the converter # but the parameter object gets the python name converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) @@ -5149,7 +5161,8 @@ def bad_node(self, node: ast.AST) -> None: self.parameter_state = ParamState.START self.function.parameters.clear() else: - fail("A 'self' parameter, if specified, must be the very first thing in the parameter block.") + fail("A 'self' parameter, if specified, must be the " + "very first thing in the parameter block.") if isinstance(converter, defining_class_converter): _lp = len(self.function.parameters) @@ -5161,16 +5174,18 @@ def bad_node(self, node: ast.AST) -> None: if self.group: fail("A 'defining_class' parameter cannot be in an optional group.") else: - fail("A 'defining_class' parameter, if specified, must either be the first thing in the parameter block, or come just after 'self'.") + fail("A 'defining_class' parameter, if specified, must either " + "be the first thing in the parameter block, or come just " + "after 'self'.") p = Parameter(parameter_name, kind, function=self.function, converter=converter, default=value, group=self.group) names = [k.name for k in self.function.parameters.values()] if parameter_name in names[1:]: - fail("You can't have two parameters named " + repr(parameter_name) + "!") + fail(f"You can't have two parameters named {parameter_name!r}!") elif names and parameter_name == names[0] and c_name is None: - fail(f"Parameter '{parameter_name}' requires a custom C name") + fail(f"Parameter {parameter_name!r} requires a custom C name") key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name self.function.parameters[key] = p @@ -5200,7 +5215,7 @@ def parse_converter( def parse_star(self, function: Function) -> None: """Parse keyword-only parameter marker '*'.""" if self.keyword_only: - fail(f"Function {function.name} uses '*' more than once.") + fail(f"Function {function.name!r} uses '*' more than once.") self.keyword_only = True def parse_opening_square_bracket(self, function: Function) -> None: @@ -5211,7 +5226,8 @@ def parse_opening_square_bracket(self, function: Function) -> None: case ParamState.REQUIRED | ParamState.GROUP_AFTER: self.parameter_state = ParamState.GROUP_AFTER case st: - fail(f"Function {function.name} has an unsupported group configuration. " + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " f"(Unexpected state {st}.b)") self.group += 1 function.docstring_only = True @@ -5219,9 +5235,9 @@ def parse_opening_square_bracket(self, function: Function) -> None: def parse_closing_square_bracket(self, function: Function) -> None: """Parse closing parameter group symbol ']'.""" if not self.group: - fail(f"Function {function.name} has a ] without a matching [.") + fail(f"Function {function.name!r} has a ']' without a matching '['.") if not any(p.group == self.group for p in function.parameters.values()): - fail(f"Function {function.name} has an empty group.\n" + fail(f"Function {function.name!r} has an empty group. " "All groups must contain at least one parameter.") self.group -= 1 match self.parameter_state: @@ -5230,13 +5246,14 @@ def parse_closing_square_bracket(self, function: Function) -> None: case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: self.parameter_state = ParamState.RIGHT_SQUARE_AFTER case st: - fail(f"Function {function.name} has an unsupported group configuration. " + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " f"(Unexpected state {st}.c)") def parse_slash(self, function: Function) -> None: """Parse positional-only parameter marker '/'.""" if self.positional_only: - fail(f"Function {function.name} uses '/' more than once.") + fail(f"Function {function.name!r} uses '/' more than once.") self.positional_only = True # REQUIRED and OPTIONAL are allowed here, that allows positional-only # without option groups to work (and have default values!) @@ -5247,10 +5264,10 @@ def parse_slash(self, function: Function) -> None: ParamState.GROUP_BEFORE, } if (self.parameter_state not in allowed) or self.group: - fail(f"Function {function.name} has an unsupported group configuration. " + fail(f"Function {function.name!r} has an unsupported group configuration. " f"(Unexpected state {self.parameter_state}.d)") if self.keyword_only: - fail(f"Function {function.name} mixes keyword-only and " + fail(f"Function {function.name!r} mixes keyword-only and " "positional-only parameters, which is unsupported.") # fixup preceding parameters for p in function.parameters.values(): @@ -5259,24 +5276,35 @@ def parse_slash(self, function: Function) -> None: if (p.kind is not inspect.Parameter.POSITIONAL_OR_KEYWORD and not isinstance(p.converter, self_converter) ): - fail(f"Function {function.name} mixes keyword-only and " + fail(f"Function {function.name!r} mixes keyword-only and " "positional-only parameters, which is unsupported.") p.kind = inspect.Parameter.POSITIONAL_ONLY - def state_parameter_docstring_start(self, line: str | None) -> None: + def state_parameter_docstring_start(self, line: str) -> None: assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" self.parameter_docstring_indent = len(self.indent.margin) assert self.indent.depth == 3 return self.next(self.state_parameter_docstring, line) + def docstring_append(self, obj: Function | Parameter, line: str) -> None: + """Add a rstripped line to the current docstring.""" + matches = re.finditer(r'[^\x00-\x7F]', line) + if offending := ", ".join([repr(m[0]) for m in matches]): + warn("Non-ascii characters are not allowed in docstrings:", + offending) + + docstring = obj.docstring + if docstring: + docstring += "\n" + if stripped := line.rstrip(): + docstring += self.indent.dedent(stripped) + obj.docstring = docstring + # every line of the docstring must start with at least F spaces, # where F > P. # these F spaces will be stripped. - def state_parameter_docstring(self, line: str | None) -> None: - assert line is not None - - stripped = line.strip() - if stripped.startswith('#'): + def state_parameter_docstring(self, line: str) -> None: + if not self.valid_line(line): return indent = self.indent.measure(line) @@ -5290,38 +5318,20 @@ def state_parameter_docstring(self, line: str | None) -> None: return self.next(self.state_function_docstring, line) assert self.function and self.function.parameters - last_parameter = next(reversed(list(self.function.parameters.values()))) - - new_docstring = last_parameter.docstring - - if new_docstring: - new_docstring += '\n' - if stripped: - new_docstring += self.indent.dedent(line) - - last_parameter.docstring = new_docstring + last_param = next(reversed(self.function.parameters.values())) + self.docstring_append(last_param, line) # the final stanza of the DSL is the docstring. - def state_function_docstring(self, line: str | None) -> None: + def state_function_docstring(self, line: str) -> None: assert self.function is not None - assert line is not None if self.group: - fail("Function " + self.function.name + " has a ] without a matching [.") + fail(f"Function {self.function.name!r} has a ']' without a matching '['.") - stripped = line.strip() - if stripped.startswith('#'): + if not self.valid_line(line): return - new_docstring = self.function.docstring - if new_docstring: - new_docstring += "\n" - if stripped: - line = self.indent.dedent(line).rstrip() - else: - line = '' - new_docstring += line - self.function.docstring = new_docstring + self.docstring_append(self.function, line) def format_docstring(self) -> str: f = self.function @@ -5543,9 +5553,9 @@ def add_parameter(text: str) -> None: if len(lines) >= 2: if lines[1]: - fail("Docstring for " + f.full_name + " does not have a summary line!\n" + - "Every non-blank function docstring must start with\n" + - "a single line summary followed by an empty line.") + fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" + "Every non-blank function docstring must start with " + "a single line summary followed by an empty line.") elif len(lines) == 1: # the docstring is only one line right now--the summary line. # add an empty line after the summary line so we have space @@ -5573,12 +5583,10 @@ def add_parameter(text: str) -> None: return docstring - def state_terminal(self, line: str | None) -> None: + def do_post_block_processing_cleanup(self) -> None: """ Called when processing the block is done. """ - assert not line - if not self.function: return @@ -5590,13 +5598,8 @@ def state_terminal(self, line: str | None) -> None: last_parameter = next(reversed(list(values))) no_parameter_after_star = last_parameter.kind != inspect.Parameter.KEYWORD_ONLY if no_parameter_after_star: - fail("Function " + self.function.name + " specifies '*' without any parameters afterwards.") - - # remove trailing whitespace from all parameter docstrings - for name, value in self.function.parameters.items(): - if not value: - continue - value.docstring = value.docstring.rstrip() + fail(f"Function {self.function.name!r} specifies '*' " + "without any parameters afterwards.") self.function.docstring = self.format_docstring() @@ -5621,10 +5624,9 @@ def state_terminal(self, line: str | None) -> None: clinic = None -def main(argv: list[str]) -> None: - import sys - import argparse +def create_cli() -> argparse.ArgumentParser: cmdline = argparse.ArgumentParser( + prog="clinic.py", description="""Preprocessor for CPython C files. The purpose of the Argument Clinic is automating all the boilerplate involved @@ -5647,14 +5649,15 @@ def main(argv: list[str]) -> None: help="the directory tree to walk in --make mode") cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*", help="the list of files to process") - ns = cmdline.parse_args(argv) + return cmdline + +def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: if ns.converters: if ns.filename: - print("Usage error: can't specify --converters and a filename at the same time.") - print() - cmdline.print_usage() - sys.exit(-1) + parser.error( + "can't specify --converters and a filename at the same time" + ) converters: list[tuple[str, str]] = [] return_converters: list[tuple[str, str]] = [] ignored = set(""" @@ -5708,19 +5711,13 @@ def main(argv: list[str]) -> None: print() print("All converters also accept (c_default=None, py_default=None, annotation=None).") print("All return converters also accept (py_default=None).") - sys.exit(0) + return if ns.make: if ns.output or ns.filename: - print("Usage error: can't use -o or filenames with --make.") - print() - cmdline.print_usage() - sys.exit(-1) + parser.error("can't use -o or filenames with --make") if not ns.srcdir: - print("Usage error: --srcdir must not be empty with --make.") - print() - cmdline.print_usage() - sys.exit(-1) + parser.error("--srcdir must not be empty with --make") for root, dirs, files in os.walk(ns.srcdir): for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): if rcs_dir in dirs: @@ -5736,14 +5733,10 @@ def main(argv: list[str]) -> None: return if not ns.filename: - cmdline.print_usage() - sys.exit(-1) + parser.error("no input files") if ns.output and len(ns.filename) > 1: - print("Usage error: can't use -o with multiple filenames.") - print() - cmdline.print_usage() - sys.exit(-1) + parser.error("can't use -o with multiple filenames") for filename in ns.filename: if ns.verbose: @@ -5751,6 +5744,17 @@ def main(argv: list[str]) -> None: parse_file(filename, output=ns.output, verify=not ns.force) +def main(argv: list[str] | None = None) -> NoReturn: + parser = create_cli() + args = parser.parse_args(argv) + try: + run_clinic(parser, args) + except ClinicError as exc: + sys.stderr.write(exc.report()) + sys.exit(1) + else: + sys.exit(0) + + if __name__ == "__main__": - main(sys.argv[1:]) - sys.exit(0) + main() diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index fbac81336b545e..876105120c97f2 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -1,7 +1,6 @@ import dataclasses as dc import re import sys -from collections.abc import Callable from typing import NoReturn @@ -44,9 +43,12 @@ def __post_init__(self) -> None: self.line_number = 0 def __repr__(self) -> str: - return ( - f"" + parts = ( + str(id(self)), + f"line={self.line_number}", + f"condition={self.condition()!r}" ) + return f"" def status(self) -> str: return str(self.line_number).rjust(4) + ": " + self.condition() @@ -66,14 +68,6 @@ def fail(self, *a: object) -> NoReturn: print(" ", ' '.join(str(x) for x in a)) sys.exit(-1) - def close(self) -> None: - if self.stack: - self.fail("Ended file while still in a preprocessor conditional block!") - - def write(self, s: str) -> None: - for line in s.split("\n"): - self.writeline(line) - def writeline(self, line: str) -> None: self.line_number += 1 line = line.strip() diff --git a/Tools/clinic/mypy.ini b/Tools/clinic/mypy.ini index 4cfc05bec01608..b1fdad673c61a1 100644 --- a/Tools/clinic/mypy.ini +++ b/Tools/clinic/mypy.ini @@ -8,5 +8,5 @@ python_version = 3.10 # and be strict! strict = True strict_concatenate = True -enable_error_code = ignore-without-code,redundant-expr +enable_error_code = ignore-without-code,redundant-expr,truthy-bool warn_unreachable = True diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 9c881897c2de1d..f798b2f772d08a 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -494,6 +494,22 @@ def calculate_object_stats(stats): rows.append((label, value, ratio)) return rows +def calculate_gc_stats(stats): + gc_stats = [] + for key, value in stats.items(): + if not key.startswith("GC"): + continue + n, _, rest = key[3:].partition("]") + name = rest.strip() + gen_n = int(n) + while len(gc_stats) <= gen_n: + gc_stats.append({}) + gc_stats[gen_n][name] = value + return [ + (i, gen["collections"], gen["objects collected"], gen["object visits"]) + for (i, gen) in enumerate(gc_stats) + ] + def emit_object_stats(stats): with Section("Object stats", summary="allocations, frees and dict materializatons"): rows = calculate_object_stats(stats) @@ -505,6 +521,22 @@ def emit_comparative_object_stats(base_stats, head_stats): head_rows = calculate_object_stats(head_stats) emit_table(("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), join_rows(base_rows, head_rows)) +def emit_gc_stats(stats): + with Section("GC stats", summary="GC collections and effectiveness"): + rows = calculate_gc_stats(stats) + emit_table(("Generation:", "Collections:", "Objects collected:", "Object visits:"), rows) + +def emit_comparative_gc_stats(base_stats, head_stats): + with Section("GC stats", summary="GC collections and effectiveness"): + base_rows = calculate_gc_stats(base_stats) + head_rows = calculate_gc_stats(head_stats) + emit_table( + ("Generation:", + "Base collections:", "Head collections:", + "Base objects collected:", "Head objects collected:", + "Base object visits:", "Head object visits:"), + join_rows(base_rows, head_rows)) + def get_total(opcode_stats): total = 0 for opcode_stat in opcode_stats: @@ -574,6 +606,7 @@ def output_single_stats(stats): emit_specialization_overview(opcode_stats, total) emit_call_stats(stats) emit_object_stats(stats) + emit_gc_stats(stats) with Section("Meta stats", summary="Meta statistics"): emit_table(("", "Count:"), [('Number of data files', stats['__nfiles__'])]) @@ -596,6 +629,7 @@ def output_comparative_stats(base_stats, head_stats): ) emit_comparative_call_stats(base_stats, head_stats) emit_comparative_object_stats(base_stats, head_stats) + emit_comparative_gc_stats(base_stats, head_stats) def output_stats(inputs, json_output=None): if len(inputs) == 1: